2024-11-14 09:43:22,488 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 09:43:22,498 main DEBUG Took 0.008300 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 09:43:22,498 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 09:43:22,498 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 09:43:22,499 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 09:43:22,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,506 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 09:43:22,517 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,519 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,520 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,521 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,522 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,523 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,523 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,524 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,525 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,526 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,526 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,527 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:43:22,527 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,528 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 09:43:22,529 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:43:22,530 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 09:43:22,532 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 09:43:22,532 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 09:43:22,533 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 09:43:22,534 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 09:43:22,540 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 09:43:22,543 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 09:43:22,544 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 09:43:22,544 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 09:43:22,545 main DEBUG createAppenders(={Console}) 2024-11-14 09:43:22,546 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 09:43:22,546 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 09:43:22,546 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 09:43:22,547 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 09:43:22,547 main DEBUG OutputStream closed 2024-11-14 09:43:22,547 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 09:43:22,547 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 09:43:22,548 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 09:43:22,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 09:43:22,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 09:43:22,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 09:43:22,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 09:43:22,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 09:43:22,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 09:43:22,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 09:43:22,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 09:43:22,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 09:43:22,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 09:43:22,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 09:43:22,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 09:43:22,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 09:43:22,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 09:43:22,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 09:43:22,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 09:43:22,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 09:43:22,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 09:43:22,624 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 09:43:22,624 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 09:43:22,625 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 09:43:22,625 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T09:43:22,823 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197 2024-11-14 09:43:22,825 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 09:43:22,826 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T09:43:22,833 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-14T09:43:22,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T09:43:22,868 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd, deleteOnExit=true 2024-11-14T09:43:22,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T09:43:22,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/test.cache.data in system properties and HBase conf 2024-11-14T09:43:22,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T09:43:22,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/hadoop.log.dir in system properties and HBase conf 2024-11-14T09:43:22,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T09:43:22,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T09:43:22,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T09:43:22,958 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T09:43:23,041 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:43:23,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:43:23,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:43:23,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T09:43:23,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:43:23,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T09:43:23,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T09:43:23,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:43:23,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:43:23,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T09:43:23,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/nfs.dump.dir in system properties and HBase conf 2024-11-14T09:43:23,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/java.io.tmpdir in system properties and HBase conf 2024-11-14T09:43:23,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:43:23,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T09:43:23,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T09:43:24,313 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T09:43:24,381 INFO [Time-limited test {}] log.Log(170): Logging initialized @2632ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T09:43:24,450 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:43:24,504 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:43:24,524 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:43:24,524 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:43:24,525 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T09:43:24,536 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:43:24,539 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:43:24,540 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:43:24,705 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/java.io.tmpdir/jetty-localhost-39611-hadoop-hdfs-3_4_1-tests_jar-_-any-13661223304291462633/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:43:24,711 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39611} 2024-11-14T09:43:24,712 INFO [Time-limited test {}] server.Server(415): Started @2963ms 2024-11-14T09:43:25,291 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:43:25,298 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:43:25,299 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:43:25,299 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:43:25,300 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T09:43:25,300 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:43:25,301 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:43:25,400 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/java.io.tmpdir/jetty-localhost-40047-hadoop-hdfs-3_4_1-tests_jar-_-any-10511772170885568611/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:43:25,401 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40047} 2024-11-14T09:43:25,401 INFO [Time-limited test {}] server.Server(415): Started @3652ms 2024-11-14T09:43:25,451 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T09:43:26,463 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/data/data1/current/BP-1223621905-172.17.0.2-1731577403708/current, will proceed with Du for space computation calculation, 2024-11-14T09:43:26,463 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/data/data2/current/BP-1223621905-172.17.0.2-1731577403708/current, will proceed with Du for space computation calculation, 2024-11-14T09:43:26,503 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T09:43:26,564 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe005208a0bb96795 with lease ID 0x884804fb5686c1e5: Processing first storage report for DS-02214576-6aa5-4b7d-be02-bf47ccd50b77 from datanode DatanodeRegistration(127.0.0.1:43879, datanodeUuid=f95f21a8-38fa-4632-ae8d-7e262c9fd601, infoPort=45257, infoSecurePort=0, ipcPort=40105, storageInfo=lv=-57;cid=testClusterID;nsid=1750113344;c=1731577403708) 2024-11-14T09:43:26,566 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe005208a0bb96795 with lease ID 0x884804fb5686c1e5: from storage DS-02214576-6aa5-4b7d-be02-bf47ccd50b77 node DatanodeRegistration(127.0.0.1:43879, datanodeUuid=f95f21a8-38fa-4632-ae8d-7e262c9fd601, infoPort=45257, infoSecurePort=0, ipcPort=40105, storageInfo=lv=-57;cid=testClusterID;nsid=1750113344;c=1731577403708), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T09:43:26,566 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe005208a0bb96795 with lease ID 0x884804fb5686c1e5: Processing first storage report for DS-6a1029dc-bd59-43f8-8e61-15d6ce6bae58 from datanode DatanodeRegistration(127.0.0.1:43879, datanodeUuid=f95f21a8-38fa-4632-ae8d-7e262c9fd601, infoPort=45257, infoSecurePort=0, ipcPort=40105, storageInfo=lv=-57;cid=testClusterID;nsid=1750113344;c=1731577403708) 2024-11-14T09:43:26,566 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe005208a0bb96795 with lease ID 0x884804fb5686c1e5: from storage DS-6a1029dc-bd59-43f8-8e61-15d6ce6bae58 node DatanodeRegistration(127.0.0.1:43879, datanodeUuid=f95f21a8-38fa-4632-ae8d-7e262c9fd601, infoPort=45257, infoSecurePort=0, ipcPort=40105, storageInfo=lv=-57;cid=testClusterID;nsid=1750113344;c=1731577403708), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T09:43:26,647 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197 2024-11-14T09:43:26,735 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/zookeeper_0, clientPort=60372, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T09:43:26,750 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60372 2024-11-14T09:43:26,768 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:43:26,772 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:43:27,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741825_1001 (size=7) 2024-11-14T09:43:27,418 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e with version=8 2024-11-14T09:43:27,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/hbase-staging 2024-11-14T09:43:27,499 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T09:43:27,738 INFO [Time-limited test {}] client.ConnectionUtils(128): master/defc576eb6b7:0 server-side Connection retries=45 2024-11-14T09:43:27,746 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:43:27,746 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:43:27,751 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:43:27,751 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:43:27,751 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:43:27,866 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T09:43:27,927 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T09:43:27,935 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T09:43:27,938 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:43:27,959 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7358 (auto-detected) 2024-11-14T09:43:27,960 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T09:43:27,976 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33347 2024-11-14T09:43:27,995 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33347 connecting to ZooKeeper ensemble=127.0.0.1:60372 2024-11-14T09:43:28,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:333470x0, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:43:28,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33347-0x10138ba60a10000 connected 2024-11-14T09:43:28,190 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:43:28,193 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:43:28,203 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:43:28,208 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e, hbase.cluster.distributed=false 2024-11-14T09:43:28,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:43:28,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33347 2024-11-14T09:43:28,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33347 2024-11-14T09:43:28,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33347 2024-11-14T09:43:28,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33347 2024-11-14T09:43:28,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33347 2024-11-14T09:43:28,328 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/defc576eb6b7:0 server-side Connection retries=45 2024-11-14T09:43:28,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:43:28,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:43:28,330 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:43:28,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:43:28,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:43:28,332 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T09:43:28,335 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:43:28,336 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42511 2024-11-14T09:43:28,338 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42511 connecting to ZooKeeper ensemble=127.0.0.1:60372 2024-11-14T09:43:28,339 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:43:28,342 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:43:28,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425110x0, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:43:28,357 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425110x0, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:43:28,357 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42511-0x10138ba60a10001 connected 2024-11-14T09:43:28,360 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T09:43:28,370 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T09:43:28,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T09:43:28,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:43:28,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42511 2024-11-14T09:43:28,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42511 2024-11-14T09:43:28,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42511 2024-11-14T09:43:28,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42511 2024-11-14T09:43:28,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42511 2024-11-14T09:43:28,393 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;defc576eb6b7:33347 2024-11-14T09:43:28,393 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/defc576eb6b7,33347,1731577407591 2024-11-14T09:43:28,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:43:28,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:43:28,412 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/defc576eb6b7,33347,1731577407591 2024-11-14T09:43:28,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T09:43:28,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:28,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:28,442 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T09:43:28,443 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/defc576eb6b7,33347,1731577407591 from backup master directory 2024-11-14T09:43:28,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/defc576eb6b7,33347,1731577407591 2024-11-14T09:43:28,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:43:28,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:43:28,452 WARN [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:43:28,452 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=defc576eb6b7,33347,1731577407591 2024-11-14T09:43:28,454 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T09:43:28,455 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T09:43:28,507 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/hbase.id] with ID: b85ece77-3b90-40fa-9bf4-8df5cb511dfe 2024-11-14T09:43:28,507 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/.tmp/hbase.id 2024-11-14T09:43:28,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741826_1002 (size=42) 2024-11-14T09:43:28,919 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/.tmp/hbase.id]:[hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/hbase.id] 2024-11-14T09:43:28,967 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:43:28,973 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T09:43:28,998 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-14T09:43:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:29,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741827_1003 (size=196) 2024-11-14T09:43:29,487 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:43:29,490 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T09:43:29,504 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:29,507 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:43:29,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741828_1004 (size=1189) 2024-11-14T09:43:29,962 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store 2024-11-14T09:43:29,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741829_1005 (size=34) 2024-11-14T09:43:30,389 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T09:43:30,394 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:43:30,395 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:43:30,395 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:43:30,396 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:43:30,398 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:43:30,398 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:43:30,398 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:43:30,399 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731577410395Disabling compacts and flushes for region at 1731577410395Disabling writes for close at 1731577410398 (+3 ms)Writing region close event to WAL at 1731577410398Closed at 1731577410398 2024-11-14T09:43:30,402 WARN [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/.initializing 2024-11-14T09:43:30,402 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/WALs/defc576eb6b7,33347,1731577407591 2024-11-14T09:43:30,409 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:43:30,423 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=defc576eb6b7%2C33347%2C1731577407591, suffix=, logDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/WALs/defc576eb6b7,33347,1731577407591, archiveDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/oldWALs, maxLogs=10 2024-11-14T09:43:30,443 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/WALs/defc576eb6b7,33347,1731577407591/defc576eb6b7%2C33347%2C1731577407591.1731577410427, exclude list is [], retry=0 2024-11-14T09:43:30,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43879,DS-02214576-6aa5-4b7d-be02-bf47ccd50b77,DISK] 2024-11-14T09:43:30,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T09:43:30,493 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/WALs/defc576eb6b7,33347,1731577407591/defc576eb6b7%2C33347%2C1731577407591.1731577410427 2024-11-14T09:43:30,494 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45257:45257)] 2024-11-14T09:43:30,494 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:43:30,495 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:43:30,497 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,498 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T09:43:30,554 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:30,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:30,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T09:43:30,560 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:30,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:43:30,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T09:43:30,564 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:30,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:43:30,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,568 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T09:43:30,568 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:30,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:43:30,570 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,573 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,574 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,579 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,580 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,583 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:43:30,587 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:43:30,592 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:43:30,593 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67539073, jitterRate=0.006410613656044006}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:43:30,600 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731577410509Initializing all the Stores at 1731577410511 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577410511Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577410512 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577410512Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577410512Cleaning up temporary data from old regions at 1731577410580 (+68 ms)Region opened successfully at 1731577410599 (+19 ms) 2024-11-14T09:43:30,601 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T09:43:30,631 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bee8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=defc576eb6b7/172.17.0.2:0 2024-11-14T09:43:30,657 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T09:43:30,667 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T09:43:30,667 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T09:43:30,670 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T09:43:30,671 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T09:43:30,675 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-14T09:43:30,675 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T09:43:30,697 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T09:43:30,706 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T09:43:30,809 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T09:43:30,813 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T09:43:30,816 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T09:43:30,830 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T09:43:30,832 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T09:43:30,837 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T09:43:30,850 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T09:43:30,853 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T09:43:30,862 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T09:43:30,885 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T09:43:30,893 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T09:43:30,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:43:30,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:43:30,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:30,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:30,907 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=defc576eb6b7,33347,1731577407591, sessionid=0x10138ba60a10000, setting cluster-up flag (Was=false) 2024-11-14T09:43:30,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:30,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:30,967 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T09:43:30,971 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=defc576eb6b7,33347,1731577407591 2024-11-14T09:43:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:31,030 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T09:43:31,033 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=defc576eb6b7,33347,1731577407591 2024-11-14T09:43:31,042 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T09:43:31,086 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(746): ClusterId : b85ece77-3b90-40fa-9bf4-8df5cb511dfe 2024-11-14T09:43:31,088 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T09:43:31,095 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T09:43:31,095 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T09:43:31,105 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T09:43:31,106 DEBUG [RS:0;defc576eb6b7:42511 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c8feae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=defc576eb6b7/172.17.0.2:0 2024-11-14T09:43:31,108 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T09:43:31,120 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T09:43:31,126 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;defc576eb6b7:42511 2024-11-14T09:43:31,128 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T09:43:31,130 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T09:43:31,130 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T09:43:31,130 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T09:43:31,133 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(2659): reportForDuty to master=defc576eb6b7,33347,1731577407591 with port=42511, startcode=1731577408298 2024-11-14T09:43:31,134 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: defc576eb6b7,33347,1731577407591 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T09:43:31,141 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:43:31,141 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:43:31,141 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:43:31,141 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:43:31,142 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/defc576eb6b7:0, corePoolSize=10, maxPoolSize=10 2024-11-14T09:43:31,142 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,142 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/defc576eb6b7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:43:31,142 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,144 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731577441144 2024-11-14T09:43:31,144 DEBUG [RS:0;defc576eb6b7:42511 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T09:43:31,146 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T09:43:31,147 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:43:31,147 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T09:43:31,147 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T09:43:31,150 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T09:43:31,151 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T09:43:31,151 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T09:43:31,151 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T09:43:31,152 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:31,152 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,153 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T09:43:31,155 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T09:43:31,156 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T09:43:31,157 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T09:43:31,159 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T09:43:31,159 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T09:43:31,162 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.large.0-1731577411160,5,FailOnTimeoutGroup] 2024-11-14T09:43:31,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741831_1007 (size=1321) 2024-11-14T09:43:31,163 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.small.0-1731577411162,5,FailOnTimeoutGroup] 2024-11-14T09:43:31,163 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,163 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T09:43:31,164 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,164 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51707, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T09:43:31,182 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33347 {}] master.ServerManager(363): Checking decommissioned status of RegionServer defc576eb6b7,42511,1731577408298 2024-11-14T09:43:31,184 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33347 {}] master.ServerManager(517): Registering regionserver=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:31,196 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:43:31,196 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35709 2024-11-14T09:43:31,196 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T09:43:31,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:43:31,209 DEBUG [RS:0;defc576eb6b7:42511 {}] zookeeper.ZKUtil(111): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/defc576eb6b7,42511,1731577408298 2024-11-14T09:43:31,209 WARN [RS:0;defc576eb6b7:42511 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:43:31,210 INFO [RS:0;defc576eb6b7:42511 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:43:31,210 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/WALs/defc576eb6b7,42511,1731577408298 2024-11-14T09:43:31,212 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [defc576eb6b7,42511,1731577408298] 2024-11-14T09:43:31,232 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T09:43:31,247 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T09:43:31,251 INFO [RS:0;defc576eb6b7:42511 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:43:31,251 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,252 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T09:43:31,256 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T09:43:31,257 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,258 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,258 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,258 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,258 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,258 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,258 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/defc576eb6b7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:43:31,258 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,259 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,259 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,259 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,259 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,259 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:43:31,259 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/defc576eb6b7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:43:31,259 DEBUG [RS:0;defc576eb6b7:42511 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:43:31,260 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,260 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,261 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,261 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,261 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,261 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,42511,1731577408298-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:43:31,277 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T09:43:31,279 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,42511,1731577408298-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,279 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,279 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.Replication(171): defc576eb6b7,42511,1731577408298 started 2024-11-14T09:43:31,295 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:31,295 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1482): Serving as defc576eb6b7,42511,1731577408298, RpcServer on defc576eb6b7/172.17.0.2:42511, sessionid=0x10138ba60a10001 2024-11-14T09:43:31,296 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T09:43:31,296 DEBUG [RS:0;defc576eb6b7:42511 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager defc576eb6b7,42511,1731577408298 2024-11-14T09:43:31,296 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'defc576eb6b7,42511,1731577408298' 2024-11-14T09:43:31,297 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T09:43:31,297 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T09:43:31,298 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T09:43:31,298 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T09:43:31,298 DEBUG [RS:0;defc576eb6b7:42511 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager defc576eb6b7,42511,1731577408298 2024-11-14T09:43:31,298 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'defc576eb6b7,42511,1731577408298' 2024-11-14T09:43:31,298 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T09:43:31,299 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T09:43:31,300 DEBUG [RS:0;defc576eb6b7:42511 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T09:43:31,300 INFO [RS:0;defc576eb6b7:42511 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T09:43:31,300 INFO [RS:0;defc576eb6b7:42511 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T09:43:31,409 INFO [RS:0;defc576eb6b7:42511 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:43:31,415 INFO [RS:0;defc576eb6b7:42511 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=defc576eb6b7%2C42511%2C1731577408298, suffix=, logDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/WALs/defc576eb6b7,42511,1731577408298, archiveDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/oldWALs, maxLogs=32 2024-11-14T09:43:31,432 DEBUG [RS:0;defc576eb6b7:42511 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/WALs/defc576eb6b7,42511,1731577408298/defc576eb6b7%2C42511%2C1731577408298.1731577411418, exclude list is [], retry=0 2024-11-14T09:43:31,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43879,DS-02214576-6aa5-4b7d-be02-bf47ccd50b77,DISK] 2024-11-14T09:43:31,440 INFO [RS:0;defc576eb6b7:42511 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/WALs/defc576eb6b7,42511,1731577408298/defc576eb6b7%2C42511%2C1731577408298.1731577411418 2024-11-14T09:43:31,441 DEBUG [RS:0;defc576eb6b7:42511 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45257:45257)] 2024-11-14T09:43:31,565 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T09:43:31,565 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:43:31,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741833_1009 (size=32) 2024-11-14T09:43:31,981 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:43:31,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:43:31,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:43:31,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:31,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:31,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:43:31,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:43:31,995 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:31,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:31,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:43:31,998 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:43:31,998 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:31,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:32,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:43:32,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:43:32,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:32,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:32,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:43:32,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740 2024-11-14T09:43:32,005 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740 2024-11-14T09:43:32,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:43:32,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:43:32,009 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:43:32,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:43:32,015 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:43:32,016 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75246059, jitterRate=0.12125365436077118}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:43:32,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731577411982Initializing all the Stores at 1731577411987 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577411987Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577411988 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577411988Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577411988Cleaning up temporary data from old regions at 1731577412008 (+20 ms)Region opened successfully at 1731577412018 (+10 ms) 2024-11-14T09:43:32,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:43:32,018 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:43:32,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:43:32,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:43:32,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:43:32,020 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:43:32,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731577412018Disabling compacts and flushes for region at 1731577412018Disabling writes for close at 1731577412018Writing region close event to WAL at 1731577412019 (+1 ms)Closed at 1731577412020 (+1 ms) 2024-11-14T09:43:32,023 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:43:32,023 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T09:43:32,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T09:43:32,038 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:43:32,042 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T09:43:32,197 DEBUG [defc576eb6b7:33347 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T09:43:32,210 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:32,217 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as defc576eb6b7,42511,1731577408298, state=OPENING 2024-11-14T09:43:32,272 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T09:43:32,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:32,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:43:32,284 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:43:32,284 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:43:32,288 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:43:32,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:43:32,471 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:43:32,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42837, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:43:32,484 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T09:43:32,484 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:43:32,485 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T09:43:32,488 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=defc576eb6b7%2C42511%2C1731577408298.meta, suffix=.meta, logDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/WALs/defc576eb6b7,42511,1731577408298, archiveDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/oldWALs, maxLogs=32 2024-11-14T09:43:32,503 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/WALs/defc576eb6b7,42511,1731577408298/defc576eb6b7%2C42511%2C1731577408298.meta.1731577412490.meta, exclude list is [], retry=0 2024-11-14T09:43:32,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43879,DS-02214576-6aa5-4b7d-be02-bf47ccd50b77,DISK] 2024-11-14T09:43:32,509 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/WALs/defc576eb6b7,42511,1731577408298/defc576eb6b7%2C42511%2C1731577408298.meta.1731577412490.meta 2024-11-14T09:43:32,510 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45257:45257)] 2024-11-14T09:43:32,510 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:43:32,511 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T09:43:32,514 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T09:43:32,518 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T09:43:32,521 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T09:43:32,522 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:43:32,522 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T09:43:32,522 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T09:43:32,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:43:32,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:43:32,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:32,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:32,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:43:32,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:43:32,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:32,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:32,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:43:32,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:43:32,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:32,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:32,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:43:32,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:43:32,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:32,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:43:32,535 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:43:32,536 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740 2024-11-14T09:43:32,539 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740 2024-11-14T09:43:32,542 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:43:32,542 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:43:32,543 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:43:32,545 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:43:32,547 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61504271, jitterRate=-0.08351494371891022}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:43:32,547 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T09:43:32,549 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731577412523Writing region info on filesystem at 1731577412523Initializing all the Stores at 1731577412525 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577412525Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577412525Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577412525Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577412525Cleaning up temporary data from old regions at 1731577412542 (+17 ms)Running coprocessor post-open hooks at 1731577412548 (+6 ms)Region opened successfully at 1731577412549 (+1 ms) 2024-11-14T09:43:32,555 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731577412464 2024-11-14T09:43:32,566 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T09:43:32,566 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T09:43:32,569 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:32,572 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as defc576eb6b7,42511,1731577408298, state=OPEN 2024-11-14T09:43:32,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:43:32,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:43:32,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:43:32,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:43:32,607 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:32,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T09:43:32,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=defc576eb6b7,42511,1731577408298 in 316 msec 2024-11-14T09:43:32,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T09:43:32,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 588 msec 2024-11-14T09:43:32,624 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:43:32,625 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T09:43:32,642 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:32,644 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:32,662 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:32,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50513, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:32,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6170 sec 2024-11-14T09:43:32,688 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731577412688, completionTime=-1 2024-11-14T09:43:32,690 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T09:43:32,690 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T09:43:32,714 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T09:43:32,714 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731577472714 2024-11-14T09:43:32,714 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731577532714 2024-11-14T09:43:32,714 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-14T09:43:32,716 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,33347,1731577407591-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:32,716 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,33347,1731577407591-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:32,717 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,33347,1731577407591-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:32,718 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-defc576eb6b7:33347, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:32,718 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:32,719 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:32,725 DEBUG [master/defc576eb6b7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T09:43:32,750 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.298sec 2024-11-14T09:43:32,751 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T09:43:32,753 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T09:43:32,754 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T09:43:32,755 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T09:43:32,755 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T09:43:32,756 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,33347,1731577407591-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:43:32,756 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,33347,1731577407591-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T09:43:32,765 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T09:43:32,766 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T09:43:32,766 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,33347,1731577407591-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:43:32,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:32,798 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T09:43:32,798 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T09:43:32,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:32,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:32,811 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:32,823 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:32,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:32,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:32,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:32,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=defc576eb6b7,33347,1731577407591 2024-11-14T09:43:32,866 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=5148 2024-11-14T09:43:32,874 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:43:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,33347,1731577407591 2024-11-14T09:43:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-11-14T09:43:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:43:32,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:43:32,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:43:32,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:43:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:32,931 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:43:32,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T09:43:32,934 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:32,937 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:43:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:43:32,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741835_1011 (size=1061) 2024-11-14T09:43:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:43:33,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:43:33,353 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:43:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741836_1012 (size=53) 2024-11-14T09:43:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:43:33,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:43:33,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 52ef1722f0e126ebd82cd709b2dcdf9b, disabling compactions & flushes 2024-11-14T09:43:33,765 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:33,765 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:33,765 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. after waiting 0 ms 2024-11-14T09:43:33,765 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:33,765 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:33,765 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 52ef1722f0e126ebd82cd709b2dcdf9b: Waiting for close lock at 1731577413764Disabling compacts and flushes for region at 1731577413764Disabling writes for close at 1731577413765 (+1 ms)Writing region close event to WAL at 1731577413765Closed at 1731577413765 2024-11-14T09:43:33,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:43:33,773 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577413768"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577413768"}]},"ts":"1731577413768"} 2024-11-14T09:43:33,780 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:43:33,783 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:43:33,786 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577413783"}]},"ts":"1731577413783"} 2024-11-14T09:43:33,792 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:43:33,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, ASSIGN}] 2024-11-14T09:43:33,798 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, ASSIGN 2024-11-14T09:43:33,801 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, ASSIGN; state=OFFLINE, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=false 2024-11-14T09:43:33,953 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=52ef1722f0e126ebd82cd709b2dcdf9b, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:33,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, ASSIGN because future has completed 2024-11-14T09:43:33,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:43:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:43:34,123 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:34,123 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:43:34,124 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,124 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:43:34,124 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,124 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,127 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,132 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:43:34,132 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52ef1722f0e126ebd82cd709b2dcdf9b columnFamilyName A 2024-11-14T09:43:34,132 DEBUG [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:34,136 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.HStore(327): Store=52ef1722f0e126ebd82cd709b2dcdf9b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:43:34,136 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,139 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:43:34,140 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52ef1722f0e126ebd82cd709b2dcdf9b columnFamilyName B 2024-11-14T09:43:34,140 DEBUG [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:34,142 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.HStore(327): Store=52ef1722f0e126ebd82cd709b2dcdf9b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:43:34,142 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,145 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:43:34,145 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52ef1722f0e126ebd82cd709b2dcdf9b columnFamilyName C 2024-11-14T09:43:34,145 DEBUG [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:34,147 INFO [StoreOpener-52ef1722f0e126ebd82cd709b2dcdf9b-1 {}] regionserver.HStore(327): Store=52ef1722f0e126ebd82cd709b2dcdf9b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:43:34,148 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:34,148 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,150 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,151 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,154 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,154 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,157 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:43:34,161 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,173 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:43:34,175 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 52ef1722f0e126ebd82cd709b2dcdf9b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64567700, jitterRate=-0.037866294384002686}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:43:34,175 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:34,178 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 52ef1722f0e126ebd82cd709b2dcdf9b: Running coprocessor pre-open hook at 1731577414125Writing region info on filesystem at 1731577414125Initializing all the Stores at 1731577414127 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577414127Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577414127Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577414127Cleaning up temporary data from old regions at 1731577414154 (+27 ms)Running coprocessor post-open hooks at 1731577414175 (+21 ms)Region opened successfully at 1731577414178 (+3 ms) 2024-11-14T09:43:34,182 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., pid=6, masterSystemTime=1731577414114 2024-11-14T09:43:34,189 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:34,189 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:34,189 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=52ef1722f0e126ebd82cd709b2dcdf9b, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:34,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:43:34,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T09:43:34,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 in 241 msec 2024-11-14T09:43:34,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T09:43:34,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, ASSIGN in 413 msec 2024-11-14T09:43:34,217 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:43:34,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577414217"}]},"ts":"1731577414217"} 2024-11-14T09:43:34,222 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:43:34,224 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:43:34,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3080 sec 2024-11-14T09:43:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:43:35,091 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:35,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea8828e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,101 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2b9a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:43:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,33347,1731577407591 2024-11-14T09:43:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1867cfde 2024-11-14T09:43:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:43:35,117 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:43:35,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,124 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,128 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,131 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,135 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,141 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,146 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,150 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee18a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256affd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,157 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,160 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,164 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dae11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,172 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1290d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4414c04e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,177 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a7f0f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:43:35,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:43:35,183 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:43:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:43:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:43:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b0f1b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:43:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:43:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:35,185 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:43:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66945757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:43:35,192 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,192 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,192 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,192 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,192 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,193 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,194 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,203 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,204 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:43:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:43:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:43:35,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:43:35,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:43:35,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:43:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T09:43:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:43:35,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:43:35,246 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:43:35,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:43:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:35,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:43:35,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:35,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:35,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:35,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:35,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:35,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:43:35,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/93910310fcec475e8a50ef5b848d2c32 is 50, key is test_row_0/A:col10/1731577415288/Put/seqid=0 2024-11-14T09:43:35,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:35,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741837_1013 (size=16681) 2024-11-14T09:43:35,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/93910310fcec475e8a50ef5b848d2c32 2024-11-14T09:43:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577475479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577475474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577475488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577475507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577475498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:43:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e7b067cf71a54f27bb13fdf5192c665a is 50, key is test_row_0/B:col10/1731577415288/Put/seqid=0 2024-11-14T09:43:35,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:35,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741838_1014 (size=12001) 2024-11-14T09:43:35,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e7b067cf71a54f27bb13fdf5192c665a 2024-11-14T09:43:35,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577475691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577475694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577475694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577475696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577475697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:35,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b24ff9d6cb084377bc82cd5812ebab96 is 50, key is test_row_0/C:col10/1731577415288/Put/seqid=0 2024-11-14T09:43:35,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741839_1015 (size=12001) 2024-11-14T09:43:35,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b24ff9d6cb084377bc82cd5812ebab96 2024-11-14T09:43:35,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/93910310fcec475e8a50ef5b848d2c32 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/93910310fcec475e8a50ef5b848d2c32 2024-11-14T09:43:35,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:35,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/93910310fcec475e8a50ef5b848d2c32, entries=250, sequenceid=13, filesize=16.3 K 2024-11-14T09:43:35,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e7b067cf71a54f27bb13fdf5192c665a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e7b067cf71a54f27bb13fdf5192c665a 2024-11-14T09:43:35,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e7b067cf71a54f27bb13fdf5192c665a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T09:43:35,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b24ff9d6cb084377bc82cd5812ebab96 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b24ff9d6cb084377bc82cd5812ebab96 2024-11-14T09:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:43:35,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b24ff9d6cb084377bc82cd5812ebab96, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T09:43:35,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 52ef1722f0e126ebd82cd709b2dcdf9b in 598ms, sequenceid=13, compaction requested=false 2024-11-14T09:43:35,900 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:43:35,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:35,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577415432, max timestamp=1731577415919Num uniques 859; 2024-11-14T09:43:35,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:35,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577415432, max timestamp=1731577415919Num uniques 859; 2024-11-14T09:43:35,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:35,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-14T09:43:35,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:35,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:35,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:35,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:35,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:35,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:35,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:35,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:35,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b50c918a1c0648f79b4054923644a0ec is 50, key is test_row_0/A:col10/1731577415496/Put/seqid=0 2024-11-14T09:43:36,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741840_1016 (size=12001) 2024-11-14T09:43:36,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b50c918a1c0648f79b4054923644a0ec 2024-11-14T09:43:36,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/7e11cdc279ad47e8a3af3096b82550a8 is 50, key is test_row_0/B:col10/1731577415496/Put/seqid=0 2024-11-14T09:43:36,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741841_1017 (size=12001) 2024-11-14T09:43:36,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/7e11cdc279ad47e8a3af3096b82550a8 2024-11-14T09:43:36,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:36,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/ec03f68ca0f74186a621b4f230d10938 is 50, key is test_row_0/C:col10/1731577415496/Put/seqid=0 2024-11-14T09:43:36,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741842_1018 (size=12001) 2024-11-14T09:43:36,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577476222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577476232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577476233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577476291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577476286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:43:36,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577476403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577476403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577476403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577476441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577476442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:36,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:36,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/ec03f68ca0f74186a621b4f230d10938 2024-11-14T09:43:36,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b50c918a1c0648f79b4054923644a0ec as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b50c918a1c0648f79b4054923644a0ec 2024-11-14T09:43:36,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577476663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b50c918a1c0648f79b4054923644a0ec, entries=150, sequenceid=35, filesize=11.7 K 2024-11-14T09:43:36,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577476663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/7e11cdc279ad47e8a3af3096b82550a8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/7e11cdc279ad47e8a3af3096b82550a8 2024-11-14T09:43:36,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/7e11cdc279ad47e8a3af3096b82550a8, entries=150, sequenceid=35, filesize=11.7 K 2024-11-14T09:43:36,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577476701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/ec03f68ca0f74186a621b4f230d10938 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ec03f68ca0f74186a621b4f230d10938 2024-11-14T09:43:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577476702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577476682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:36,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ec03f68ca0f74186a621b4f230d10938, entries=150, sequenceid=35, filesize=11.7 K 2024-11-14T09:43:36,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 52ef1722f0e126ebd82cd709b2dcdf9b in 768ms, sequenceid=35, compaction requested=false 2024-11-14T09:43:36,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:36,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:43:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:36,796 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:43:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:36,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:36,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/d3e15164e91f4d7ea45697657964920f is 50, key is test_row_0/A:col10/1731577416267/Put/seqid=0 2024-11-14T09:43:36,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741843_1019 (size=12001) 2024-11-14T09:43:36,849 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/d3e15164e91f4d7ea45697657964920f 2024-11-14T09:43:36,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/51a5b2c7c836456b9e6a01f1e4c2e950 is 50, key is test_row_0/B:col10/1731577416267/Put/seqid=0 2024-11-14T09:43:36,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741844_1020 (size=12001) 2024-11-14T09:43:36,927 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/51a5b2c7c836456b9e6a01f1e4c2e950 2024-11-14T09:43:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/7d0bedf2ea7e43f69de157528206dc7a is 50, key is test_row_0/C:col10/1731577416267/Put/seqid=0 2024-11-14T09:43:36,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741845_1021 (size=12001) 2024-11-14T09:43:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:36,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:37,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577477024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577477031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577477033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577477036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577477040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577477142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577477152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577477152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577477152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,227 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T09:43:37,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577477362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577477362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577477363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577477363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,388 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/7d0bedf2ea7e43f69de157528206dc7a 2024-11-14T09:43:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:43:37,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/d3e15164e91f4d7ea45697657964920f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/d3e15164e91f4d7ea45697657964920f 2024-11-14T09:43:37,430 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/d3e15164e91f4d7ea45697657964920f, entries=150, sequenceid=55, filesize=11.7 K 2024-11-14T09:43:37,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/51a5b2c7c836456b9e6a01f1e4c2e950 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/51a5b2c7c836456b9e6a01f1e4c2e950 2024-11-14T09:43:37,448 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/51a5b2c7c836456b9e6a01f1e4c2e950, entries=150, sequenceid=55, filesize=11.7 K 2024-11-14T09:43:37,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/7d0bedf2ea7e43f69de157528206dc7a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7d0bedf2ea7e43f69de157528206dc7a 2024-11-14T09:43:37,479 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7d0bedf2ea7e43f69de157528206dc7a, entries=150, sequenceid=55, filesize=11.7 K 2024-11-14T09:43:37,482 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 52ef1722f0e126ebd82cd709b2dcdf9b in 688ms, sequenceid=55, compaction requested=true 2024-11-14T09:43:37,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:37,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:37,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T09:43:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T09:43:37,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T09:43:37,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2380 sec 2024-11-14T09:43:37,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2760 sec 2024-11-14T09:43:37,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:43:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:37,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/e143070bc81f4db79c4a15a13dd06a17 is 50, key is test_row_0/A:col10/1731577417019/Put/seqid=0 2024-11-14T09:43:37,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741846_1022 (size=12001) 2024-11-14T09:43:37,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577477698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577477700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577477708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577477712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577477717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577477821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577477831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577477831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577477832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577477831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/e143070bc81f4db79c4a15a13dd06a17 2024-11-14T09:43:38,037 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:43:38,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e793f711ee9a458cb1c1b895fb2e123d is 50, key is test_row_0/B:col10/1731577417019/Put/seqid=0 2024-11-14T09:43:38,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577478040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577478051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577478052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577478052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577478052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741847_1023 (size=12001) 2024-11-14T09:43:38,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e793f711ee9a458cb1c1b895fb2e123d 2024-11-14T09:43:38,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8105d63eca334d1ca5d5026cd4df76db is 50, key is test_row_0/C:col10/1731577417019/Put/seqid=0 2024-11-14T09:43:38,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741848_1024 (size=12001) 2024-11-14T09:43:38,163 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T09:43:38,164 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T09:43:38,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:43:38,168 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T09:43:38,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T09:43:38,168 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T09:43:38,170 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T09:43:38,170 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T09:43:38,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577478362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577478373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577478374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577478374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577478374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8105d63eca334d1ca5d5026cd4df76db 2024-11-14T09:43:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/e143070bc81f4db79c4a15a13dd06a17 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e143070bc81f4db79c4a15a13dd06a17 2024-11-14T09:43:38,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e143070bc81f4db79c4a15a13dd06a17, entries=150, sequenceid=68, filesize=11.7 K 2024-11-14T09:43:38,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e793f711ee9a458cb1c1b895fb2e123d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e793f711ee9a458cb1c1b895fb2e123d 2024-11-14T09:43:38,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e793f711ee9a458cb1c1b895fb2e123d, entries=150, sequenceid=68, filesize=11.7 K 2024-11-14T09:43:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8105d63eca334d1ca5d5026cd4df76db as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8105d63eca334d1ca5d5026cd4df76db 2024-11-14T09:43:38,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8105d63eca334d1ca5d5026cd4df76db, entries=150, sequenceid=68, filesize=11.7 K 2024-11-14T09:43:38,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1060ms, sequenceid=68, compaction requested=true 2024-11-14T09:43:38,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:38,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:38,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:38,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:38,655 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:38,655 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:38,661 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:38,661 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:38,663 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:38,663 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:38,663 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:38,663 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:38,664 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e7b067cf71a54f27bb13fdf5192c665a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/7e11cdc279ad47e8a3af3096b82550a8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/51a5b2c7c836456b9e6a01f1e4c2e950, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e793f711ee9a458cb1c1b895fb2e123d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=46.9 K 2024-11-14T09:43:38,664 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/93910310fcec475e8a50ef5b848d2c32, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b50c918a1c0648f79b4054923644a0ec, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/d3e15164e91f4d7ea45697657964920f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e143070bc81f4db79c4a15a13dd06a17] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=51.4 K 2024-11-14T09:43:38,666 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93910310fcec475e8a50ef5b848d2c32, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577415268 2024-11-14T09:43:38,667 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b067cf71a54f27bb13fdf5192c665a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577415269 2024-11-14T09:43:38,667 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b50c918a1c0648f79b4054923644a0ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731577415488 2024-11-14T09:43:38,667 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3e15164e91f4d7ea45697657964920f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577416177 2024-11-14T09:43:38,668 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e143070bc81f4db79c4a15a13dd06a17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731577417006 2024-11-14T09:43:38,668 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e11cdc279ad47e8a3af3096b82550a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731577415488 2024-11-14T09:43:38,670 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a5b2c7c836456b9e6a01f1e4c2e950, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577416177 2024-11-14T09:43:38,672 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e793f711ee9a458cb1c1b895fb2e123d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731577417006 2024-11-14T09:43:38,718 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:38,719 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/168d01899246428d8c7c6f690260f928 is 50, key is test_row_0/A:col10/1731577417019/Put/seqid=0 2024-11-14T09:43:38,731 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#13 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:38,732 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ce7407afbe37403bbb26dc45f303efb6 is 50, key is test_row_0/B:col10/1731577417019/Put/seqid=0 2024-11-14T09:43:38,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741850_1026 (size=12139) 2024-11-14T09:43:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741849_1025 (size=12139) 2024-11-14T09:43:38,795 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ce7407afbe37403bbb26dc45f303efb6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ce7407afbe37403bbb26dc45f303efb6 2024-11-14T09:43:38,798 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/168d01899246428d8c7c6f690260f928 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/168d01899246428d8c7c6f690260f928 2024-11-14T09:43:38,822 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into ce7407afbe37403bbb26dc45f303efb6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:38,822 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:38,822 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 168d01899246428d8c7c6f690260f928(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:38,823 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:38,825 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=12, startTime=1731577418653; duration=0sec 2024-11-14T09:43:38,825 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=12, startTime=1731577418625; duration=0sec 2024-11-14T09:43:38,826 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:38,826 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:38,826 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:38,826 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:38,826 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:38,831 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:38,831 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:38,831 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:38,831 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b24ff9d6cb084377bc82cd5812ebab96, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ec03f68ca0f74186a621b4f230d10938, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7d0bedf2ea7e43f69de157528206dc7a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8105d63eca334d1ca5d5026cd4df76db] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=46.9 K 2024-11-14T09:43:38,832 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24ff9d6cb084377bc82cd5812ebab96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577415269 2024-11-14T09:43:38,834 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec03f68ca0f74186a621b4f230d10938, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731577415488 2024-11-14T09:43:38,834 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d0bedf2ea7e43f69de157528206dc7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577416177 2024-11-14T09:43:38,835 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8105d63eca334d1ca5d5026cd4df76db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731577417006 2024-11-14T09:43:38,879 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#14 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:38,879 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/0ee0a25198044d378cec1b91f39821ac is 50, key is test_row_0/C:col10/1731577417019/Put/seqid=0 2024-11-14T09:43:38,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741851_1027 (size=12139) 2024-11-14T09:43:38,904 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/0ee0a25198044d378cec1b91f39821ac as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/0ee0a25198044d378cec1b91f39821ac 2024-11-14T09:43:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:38,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T09:43:38,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577417694, max timestamp=1731577418887Num uniques 859; 2024-11-14T09:43:38,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:38,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:38,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577417694, max timestamp=1731577418896Num uniques 859; 2024-11-14T09:43:38,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:38,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:38,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:38,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:38,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:38,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:38,925 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into 0ee0a25198044d378cec1b91f39821ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:38,926 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:38,926 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=12, startTime=1731577418654; duration=0sec 2024-11-14T09:43:38,926 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:38,926 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:38,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c2b67d375c404500b55a31de35f890e1 is 50, key is test_row_0/A:col10/1731577418897/Put/seqid=0 2024-11-14T09:43:38,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741852_1028 (size=12001) 2024-11-14T09:43:38,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c2b67d375c404500b55a31de35f890e1 2024-11-14T09:43:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577478969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577478973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ef304ea3795a4e4791cadc67e62a7509 is 50, key is test_row_0/B:col10/1731577418897/Put/seqid=0 2024-11-14T09:43:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577478977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577478979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577478980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741853_1029 (size=12001) 2024-11-14T09:43:39,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577479090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577479090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577479101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577479101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577479111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577479300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577479311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577479321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577479321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577479322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:43:39,401 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:39,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:43:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T09:43:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:43:39,407 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:43:39,408 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:43:39,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:43:39,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ef304ea3795a4e4791cadc67e62a7509 2024-11-14T09:43:39,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b8bf714cb7234ebaa4a7b151be6a68dd is 50, key is test_row_0/C:col10/1731577418897/Put/seqid=0 2024-11-14T09:43:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:43:39,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741854_1030 (size=12001) 2024-11-14T09:43:39,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b8bf714cb7234ebaa4a7b151be6a68dd 2024-11-14T09:43:39,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c2b67d375c404500b55a31de35f890e1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c2b67d375c404500b55a31de35f890e1 2024-11-14T09:43:39,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c2b67d375c404500b55a31de35f890e1, entries=150, sequenceid=94, filesize=11.7 K 2024-11-14T09:43:39,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ef304ea3795a4e4791cadc67e62a7509 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ef304ea3795a4e4791cadc67e62a7509 2024-11-14T09:43:39,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ef304ea3795a4e4791cadc67e62a7509, entries=150, sequenceid=94, filesize=11.7 K 2024-11-14T09:43:39,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:39,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:39,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:39,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:39,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b8bf714cb7234ebaa4a7b151be6a68dd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b8bf714cb7234ebaa4a7b151be6a68dd 2024-11-14T09:43:39,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b8bf714cb7234ebaa4a7b151be6a68dd, entries=150, sequenceid=94, filesize=11.7 K 2024-11-14T09:43:39,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=114.05 KB/116790 for 52ef1722f0e126ebd82cd709b2dcdf9b in 673ms, sequenceid=94, compaction requested=false 2024-11-14T09:43:39,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:39,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577418964, max timestamp=1731577419617Num uniques 859; 2024-11-14T09:43:39,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:39,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577418964, max timestamp=1731577419617Num uniques 859; 2024-11-14T09:43:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:39,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:39,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-14T09:43:39,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577418964, max timestamp=1731577419617Num uniques 859; 2024-11-14T09:43:39,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:39,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c4d87311f92e43aa8a2450666f561453 is 50, key is test_row_0/A:col10/1731577418976/Put/seqid=0 2024-11-14T09:43:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741855_1031 (size=12001) 2024-11-14T09:43:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:43:39,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:39,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577419658, max timestamp=1731577419725Num uniques 859; 2024-11-14T09:43:39,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577419658, max timestamp=1731577419725Num uniques 859; 2024-11-14T09:43:39,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:39,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577419658, max timestamp=1731577419721Num uniques 859; 2024-11-14T09:43:39,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:39,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731577419747, max timestamp=1731577419789Num uniques 1018; 2024-11-14T09:43:39,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:43:39,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731577419747, max timestamp=1731577419789Num uniques 1018; 2024-11-14T09:43:39,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731577419747, max timestamp=1731577419789Num uniques 1018; 2024-11-14T09:43:39,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:43:39,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:43:39,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:39,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:39,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:39,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:39,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:39,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731577419855, max timestamp=1731577419883Num uniques 1027; 2024-11-14T09:43:39,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731577419855, max timestamp=1731577419883Num uniques 1027; 2024-11-14T09:43:39,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731577419855, max timestamp=1731577419883Num uniques 1027; 2024-11-14T09:43:39,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T09:43:39,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T09:43:39,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T09:43:40,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731577419979, max timestamp=1731577420014Num uniques 1086; 2024-11-14T09:43:40,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-14T09:43:40,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731577419979, max timestamp=1731577420014Num uniques 1086; 2024-11-14T09:43:40,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731577419979, max timestamp=1731577420014Num uniques 1086; 2024-11-14T09:43:40,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-14T09:43:40,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-14T09:43:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:43:40,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731577420085, max timestamp=1731577420103Num uniques 1145; 2024-11-14T09:43:40,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731577420085, max timestamp=1731577420103Num uniques 1145; 2024-11-14T09:43:40,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-14T09:43:40,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-14T09:43:40,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731577420085, max timestamp=1731577420103Num uniques 1145; 2024-11-14T09:43:40,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-14T09:43:40,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c4d87311f92e43aa8a2450666f561453 2024-11-14T09:43:40,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/f9ba109845864ce792cd3fadef560be7 is 50, key is test_row_0/B:col10/1731577418976/Put/seqid=0 2024-11-14T09:43:40,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741856_1032 (size=12001) 2024-11-14T09:43:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731577420121, max timestamp=1731577420163Num uniques 1104; 2024-11-14T09:43:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731577420121, max timestamp=1731577420163Num uniques 1104; 2024-11-14T09:43:40,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/f9ba109845864ce792cd3fadef560be7 2024-11-14T09:43:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-14T09:43:40,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-14T09:43:40,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731577420121, max timestamp=1731577420163Num uniques 1104; 2024-11-14T09:43:40,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-14T09:43:40,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/5516335bb87e49fba8c4fa60777b854d is 50, key is test_row_0/C:col10/1731577418976/Put/seqid=0 2024-11-14T09:43:40,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:40,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741857_1033 (size=12001) 2024-11-14T09:43:40,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/5516335bb87e49fba8c4fa60777b854d 2024-11-14T09:43:40,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c4d87311f92e43aa8a2450666f561453 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c4d87311f92e43aa8a2450666f561453 2024-11-14T09:43:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577480220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577480220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731577420210, max timestamp=1731577420233Num uniques 1163; 2024-11-14T09:43:40,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-11-14T09:43:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577480216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731577420210, max timestamp=1731577420233Num uniques 1163; 2024-11-14T09:43:40,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=6870, suffix heap size=148920,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=150 2024-11-14T09:43:40,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c4d87311f92e43aa8a2450666f561453, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T09:43:40,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731577420210, max timestamp=1731577420233Num uniques 1163; 2024-11-14T09:43:40,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=6870, suffix heap size=148920,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=150 2024-11-14T09:43:40,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/f9ba109845864ce792cd3fadef560be7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9ba109845864ce792cd3fadef560be7 2024-11-14T09:43:40,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9ba109845864ce792cd3fadef560be7, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T09:43:40,262 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-14T09:43:40,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/5516335bb87e49fba8c4fa60777b854d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5516335bb87e49fba8c4fa60777b854d 2024-11-14T09:43:40,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5516335bb87e49fba8c4fa60777b854d, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T09:43:40,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=92.78 KB/95004 for 52ef1722f0e126ebd82cd709b2dcdf9b in 662ms, sequenceid=115, compaction requested=true 2024-11-14T09:43:40,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:40,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:40,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:40,292 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:40,292 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:40,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:40,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:40,296 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:40,297 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:40,297 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,297 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/168d01899246428d8c7c6f690260f928, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c2b67d375c404500b55a31de35f890e1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c4d87311f92e43aa8a2450666f561453] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=35.3 K 2024-11-14T09:43:40,297 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:40,298 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:40,298 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,298 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ce7407afbe37403bbb26dc45f303efb6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ef304ea3795a4e4791cadc67e62a7509, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9ba109845864ce792cd3fadef560be7] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=35.3 K 2024-11-14T09:43:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:40,300 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7407afbe37403bbb26dc45f303efb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731577417006 2024-11-14T09:43:40,302 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ef304ea3795a4e4791cadc67e62a7509, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731577417707 2024-11-14T09:43:40,302 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 168d01899246428d8c7c6f690260f928, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731577417006 2024-11-14T09:43:40,303 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ba109845864ce792cd3fadef560be7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577418972 2024-11-14T09:43:40,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=99.49 KB heapSize=282.36 KB 2024-11-14T09:43:40,305 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b67d375c404500b55a31de35f890e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731577417707 2024-11-14T09:43:40,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:40,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:40,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:40,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:40,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:40,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:40,308 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d87311f92e43aa8a2450666f561453, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577418972 2024-11-14T09:43:40,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/088c5a2d1708445d968cd698578b1f72 is 50, key is test_row_0/A:col10/1731577420267/Put/seqid=0 2024-11-14T09:43:40,354 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:40,355 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/48b9f73a937d46e2831198e27e80430a is 50, key is test_row_0/B:col10/1731577418976/Put/seqid=0 2024-11-14T09:43:40,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,360 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,361 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/e67b10970add4ccfb85da9488d18ddcb is 50, key is test_row_0/A:col10/1731577418976/Put/seqid=0 2024-11-14T09:43:40,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577480369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577480369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577480371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577480373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577480379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741858_1034 (size=14541) 2024-11-14T09:43:40,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/088c5a2d1708445d968cd698578b1f72 2024-11-14T09:43:40,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741859_1035 (size=12241) 2024-11-14T09:43:40,412 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/48b9f73a937d46e2831198e27e80430a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48b9f73a937d46e2831198e27e80430a 2024-11-14T09:43:40,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741860_1036 (size=12241) 2024-11-14T09:43:40,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4eaefec4a1a44ee28d748e71c0d5dd9e is 50, key is test_row_0/B:col10/1731577420267/Put/seqid=0 2024-11-14T09:43:40,425 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into 48b9f73a937d46e2831198e27e80430a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:40,425 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:40,425 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=13, startTime=1731577420292; duration=0sec 2024-11-14T09:43:40,426 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:40,426 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:40,426 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:40,428 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:40,429 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:40,429 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,429 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/0ee0a25198044d378cec1b91f39821ac, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b8bf714cb7234ebaa4a7b151be6a68dd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5516335bb87e49fba8c4fa60777b854d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=35.3 K 2024-11-14T09:43:40,430 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee0a25198044d378cec1b91f39821ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731577417006 2024-11-14T09:43:40,432 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b8bf714cb7234ebaa4a7b151be6a68dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731577417707 2024-11-14T09:43:40,433 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 5516335bb87e49fba8c4fa60777b854d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577418972 2024-11-14T09:43:40,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741861_1037 (size=12151) 2024-11-14T09:43:40,454 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#25 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:40,454 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/bd27f9875dc047cf9bcf3551f38e326d is 50, key is test_row_0/C:col10/1731577418976/Put/seqid=0 2024-11-14T09:43:40,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.21 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4eaefec4a1a44ee28d748e71c0d5dd9e 2024-11-14T09:43:40,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741862_1038 (size=12241) 2024-11-14T09:43:40,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/9ebee544f03645eb84f4561040c4bccc is 50, key is test_row_0/C:col10/1731577420267/Put/seqid=0 2024-11-14T09:43:40,489 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/bd27f9875dc047cf9bcf3551f38e326d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/bd27f9875dc047cf9bcf3551f38e326d 2024-11-14T09:43:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577480491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577480491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577480491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577480492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577480492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,505 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into bd27f9875dc047cf9bcf3551f38e326d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,505 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:40,505 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=13, startTime=1731577420293; duration=0sec 2024-11-14T09:43:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,505 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,505 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:40,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:40,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741863_1039 (size=12151) 2024-11-14T09:43:40,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:40,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.21 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/9ebee544f03645eb84f4561040c4bccc 2024-11-14T09:43:40,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/088c5a2d1708445d968cd698578b1f72 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/088c5a2d1708445d968cd698578b1f72 2024-11-14T09:43:40,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/088c5a2d1708445d968cd698578b1f72, entries=200, sequenceid=251, filesize=14.2 K 2024-11-14T09:43:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:43:40,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4eaefec4a1a44ee28d748e71c0d5dd9e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4eaefec4a1a44ee28d748e71c0d5dd9e 2024-11-14T09:43:40,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4eaefec4a1a44ee28d748e71c0d5dd9e, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T09:43:40,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/9ebee544f03645eb84f4561040c4bccc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9ebee544f03645eb84f4561040c4bccc 2024-11-14T09:43:40,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9ebee544f03645eb84f4561040c4bccc, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T09:43:40,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~106.20 KB/108744, heapSize ~302.23 KB/309488, currentSize=73.80 KB/75570 for 52ef1722f0e126ebd82cd709b2dcdf9b in 284ms, sequenceid=251, compaction requested=false 2024-11-14T09:43:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:40,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:43:40,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:40,669 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:43:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:40,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/939af23c53c24e3f9485f403d74855d7 is 50, key is test_row_0/A:col10/1731577420364/Put/seqid=0 2024-11-14T09:43:40,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:40,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741864_1040 (size=12301) 2024-11-14T09:43:40,724 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/939af23c53c24e3f9485f403d74855d7 2024-11-14T09:43:40,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/a2a155e3e5eb4e68ae9abf10d8a92b71 is 50, key is test_row_0/B:col10/1731577420364/Put/seqid=0 2024-11-14T09:43:40,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577480738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577480742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577480743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577480748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577480752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741865_1041 (size=12301) 2024-11-14T09:43:40,828 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/e67b10970add4ccfb85da9488d18ddcb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e67b10970add4ccfb85da9488d18ddcb 2024-11-14T09:43:40,858 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into e67b10970add4ccfb85da9488d18ddcb(size=12.0 K), total size for store is 26.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:40,858 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:40,858 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=13, startTime=1731577420292; duration=0sec 2024-11-14T09:43:40,858 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:40,858 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:40,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577480861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577480862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577480861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577480864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:40,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577480871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577481081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577481080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577481081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577481081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577481090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,182 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/a2a155e3e5eb4e68ae9abf10d8a92b71 2024-11-14T09:43:41,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/82114857f6694099be97e8208d0b08bb is 50, key is test_row_0/C:col10/1731577420364/Put/seqid=0 2024-11-14T09:43:41,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741866_1042 (size=12301) 2024-11-14T09:43:41,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577481391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577481401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577481402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577481402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577481402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:43:41,649 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/82114857f6694099be97e8208d0b08bb 2024-11-14T09:43:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/939af23c53c24e3f9485f403d74855d7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/939af23c53c24e3f9485f403d74855d7 2024-11-14T09:43:41,670 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/939af23c53c24e3f9485f403d74855d7, entries=150, sequenceid=267, filesize=12.0 K 2024-11-14T09:43:41,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/a2a155e3e5eb4e68ae9abf10d8a92b71 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/a2a155e3e5eb4e68ae9abf10d8a92b71 2024-11-14T09:43:41,691 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/a2a155e3e5eb4e68ae9abf10d8a92b71, entries=150, sequenceid=267, filesize=12.0 K 2024-11-14T09:43:41,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/82114857f6694099be97e8208d0b08bb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/82114857f6694099be97e8208d0b08bb 2024-11-14T09:43:41,705 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/82114857f6694099be97e8208d0b08bb, entries=150, sequenceid=267, filesize=12.0 K 2024-11-14T09:43:41,710 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1042ms, sequenceid=267, compaction requested=true 2024-11-14T09:43:41,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:41,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:41,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T09:43:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T09:43:41,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T09:43:41,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3050 sec 2024-11-14T09:43:41,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.3180 sec 2024-11-14T09:43:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:41,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:43:41,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:41,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:41,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:41,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:41,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:41,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:41,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/940c5e78ca6b4201b1a4181713053d17 is 50, key is test_row_0/A:col10/1731577420749/Put/seqid=0 2024-11-14T09:43:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577481934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577481936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577481938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577481946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577481947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741867_1043 (size=14741) 2024-11-14T09:43:41,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/940c5e78ca6b4201b1a4181713053d17 2024-11-14T09:43:41,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4fd7805e984949bdab5a34e69e52aadb is 50, key is test_row_0/B:col10/1731577420749/Put/seqid=0 2024-11-14T09:43:42,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741868_1044 (size=12301) 2024-11-14T09:43:42,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4fd7805e984949bdab5a34e69e52aadb 2024-11-14T09:43:42,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/2059c498de5642cdbf9a3495e2283b75 is 50, key is test_row_0/C:col10/1731577420749/Put/seqid=0 2024-11-14T09:43:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577482050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577482061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577482061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577482061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577482061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741869_1045 (size=12301) 2024-11-14T09:43:42,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577482270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577482271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577482271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577482281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577482284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/2059c498de5642cdbf9a3495e2283b75 2024-11-14T09:43:42,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/940c5e78ca6b4201b1a4181713053d17 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/940c5e78ca6b4201b1a4181713053d17 2024-11-14T09:43:42,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/940c5e78ca6b4201b1a4181713053d17, entries=200, sequenceid=287, filesize=14.4 K 2024-11-14T09:43:42,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4fd7805e984949bdab5a34e69e52aadb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4fd7805e984949bdab5a34e69e52aadb 2024-11-14T09:43:42,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4fd7805e984949bdab5a34e69e52aadb, entries=150, sequenceid=287, filesize=12.0 K 2024-11-14T09:43:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/2059c498de5642cdbf9a3495e2283b75 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/2059c498de5642cdbf9a3495e2283b75 2024-11-14T09:43:42,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/2059c498de5642cdbf9a3495e2283b75, entries=150, sequenceid=287, filesize=12.0 K 2024-11-14T09:43:42,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 52ef1722f0e126ebd82cd709b2dcdf9b in 622ms, sequenceid=287, compaction requested=true 2024-11-14T09:43:42,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:42,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:42,539 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:42,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:42,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:42,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:42,540 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:42,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:42,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:42,545 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:42,545 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53824 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:42,545 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:42,545 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:42,545 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:42,545 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:42,545 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48b9f73a937d46e2831198e27e80430a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4eaefec4a1a44ee28d748e71c0d5dd9e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/a2a155e3e5eb4e68ae9abf10d8a92b71, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4fd7805e984949bdab5a34e69e52aadb] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=47.8 K 2024-11-14T09:43:42,545 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e67b10970add4ccfb85da9488d18ddcb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/088c5a2d1708445d968cd698578b1f72, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/939af23c53c24e3f9485f403d74855d7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/940c5e78ca6b4201b1a4181713053d17] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=52.6 K 2024-11-14T09:43:42,546 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b9f73a937d46e2831198e27e80430a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577418972 2024-11-14T09:43:42,546 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e67b10970add4ccfb85da9488d18ddcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577418972 2024-11-14T09:43:42,547 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eaefec4a1a44ee28d748e71c0d5dd9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731577420267 2024-11-14T09:43:42,548 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 088c5a2d1708445d968cd698578b1f72, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731577420267 2024-11-14T09:43:42,548 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a2a155e3e5eb4e68ae9abf10d8a92b71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731577420364 2024-11-14T09:43:42,548 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939af23c53c24e3f9485f403d74855d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731577420364 2024-11-14T09:43:42,549 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940c5e78ca6b4201b1a4181713053d17, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731577420738 2024-11-14T09:43:42,549 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd7805e984949bdab5a34e69e52aadb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731577420738 2024-11-14T09:43:42,576 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:42,577 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/d8c357222ad5451d88892d44994a41f7 is 50, key is test_row_0/B:col10/1731577420749/Put/seqid=0 2024-11-14T09:43:42,577 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:42,578 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/541b677fe22b43278d4646a71a367fe0 is 50, key is test_row_0/A:col10/1731577420749/Put/seqid=0 2024-11-14T09:43:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:42,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:43:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:42,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741871_1047 (size=12677) 2024-11-14T09:43:42,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741870_1046 (size=12677) 2024-11-14T09:43:42,607 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/541b677fe22b43278d4646a71a367fe0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/541b677fe22b43278d4646a71a367fe0 2024-11-14T09:43:42,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/95d164ef01134f76964cb2265cd90e53 is 50, key is test_row_0/A:col10/1731577422586/Put/seqid=0 2024-11-14T09:43:42,626 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/d8c357222ad5451d88892d44994a41f7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d8c357222ad5451d88892d44994a41f7 2024-11-14T09:43:42,628 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 541b677fe22b43278d4646a71a367fe0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:42,628 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:42,628 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=12, startTime=1731577422539; duration=0sec 2024-11-14T09:43:42,628 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:42,628 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:42,628 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:42,634 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:42,634 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:42,634 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:42,634 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/bd27f9875dc047cf9bcf3551f38e326d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9ebee544f03645eb84f4561040c4bccc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/82114857f6694099be97e8208d0b08bb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/2059c498de5642cdbf9a3495e2283b75] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=47.8 K 2024-11-14T09:43:42,636 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd27f9875dc047cf9bcf3551f38e326d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577418972 2024-11-14T09:43:42,636 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ebee544f03645eb84f4561040c4bccc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731577420267 2024-11-14T09:43:42,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741872_1048 (size=14741) 2024-11-14T09:43:42,637 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82114857f6694099be97e8208d0b08bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731577420364 2024-11-14T09:43:42,637 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2059c498de5642cdbf9a3495e2283b75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731577420738 2024-11-14T09:43:42,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577482620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577482625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577482630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/95d164ef01134f76964cb2265cd90e53 2024-11-14T09:43:42,645 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into d8c357222ad5451d88892d44994a41f7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:42,645 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:42,645 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=12, startTime=1731577422540; duration=0sec 2024-11-14T09:43:42,646 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:42,646 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577482636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577482639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/f9459d068d2f40f5a3f15cfe7c52903b is 50, key is test_row_0/B:col10/1731577422586/Put/seqid=0 2024-11-14T09:43:42,678 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:42,679 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/35f85c7ea70b4eb9a9c917b87d6d7921 is 50, key is test_row_0/C:col10/1731577420749/Put/seqid=0 2024-11-14T09:43:42,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741873_1049 (size=12301) 2024-11-14T09:43:42,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/f9459d068d2f40f5a3f15cfe7c52903b 2024-11-14T09:43:42,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741874_1050 (size=12677) 2024-11-14T09:43:42,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/1fdf9957698f42ca887fa471c7a2aecf is 50, key is test_row_0/C:col10/1731577422586/Put/seqid=0 2024-11-14T09:43:42,724 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/35f85c7ea70b4eb9a9c917b87d6d7921 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/35f85c7ea70b4eb9a9c917b87d6d7921 2024-11-14T09:43:42,735 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into 35f85c7ea70b4eb9a9c917b87d6d7921(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:42,735 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:42,735 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=12, startTime=1731577422540; duration=0sec 2024-11-14T09:43:42,736 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:42,736 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:42,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741875_1051 (size=12301) 2024-11-14T09:43:42,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/1fdf9957698f42ca887fa471c7a2aecf 2024-11-14T09:43:42,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577482751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577482756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577482756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577482751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577482761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:42,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/95d164ef01134f76964cb2265cd90e53 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/95d164ef01134f76964cb2265cd90e53 2024-11-14T09:43:42,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/95d164ef01134f76964cb2265cd90e53, entries=200, sequenceid=303, filesize=14.4 K 2024-11-14T09:43:42,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/f9459d068d2f40f5a3f15cfe7c52903b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9459d068d2f40f5a3f15cfe7c52903b 2024-11-14T09:43:42,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9459d068d2f40f5a3f15cfe7c52903b, entries=150, sequenceid=303, filesize=12.0 K 2024-11-14T09:43:42,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/1fdf9957698f42ca887fa471c7a2aecf as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/1fdf9957698f42ca887fa471c7a2aecf 2024-11-14T09:43:42,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/1fdf9957698f42ca887fa471c7a2aecf, entries=150, sequenceid=303, filesize=12.0 K 2024-11-14T09:43:42,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 52ef1722f0e126ebd82cd709b2dcdf9b in 231ms, sequenceid=303, compaction requested=false 2024-11-14T09:43:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:42,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:43:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:42,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:42,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:42,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:42,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:43,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/97f8c77027504c75b36b4f49eb9656eb is 50, key is test_row_0/A:col10/1731577422613/Put/seqid=0 2024-11-14T09:43:43,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577483004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577483005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577483008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577483010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577483012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741876_1052 (size=14741) 2024-11-14T09:43:43,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/97f8c77027504c75b36b4f49eb9656eb 2024-11-14T09:43:43,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/46e2ed46f473414dac0e4379370d5c0d is 50, key is test_row_0/B:col10/1731577422613/Put/seqid=0 2024-11-14T09:43:43,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741877_1053 (size=12301) 2024-11-14T09:43:43,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/46e2ed46f473414dac0e4379370d5c0d 2024-11-14T09:43:43,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c621da6b3cf14564ae62ce06f788b1e4 is 50, key is test_row_0/C:col10/1731577422613/Put/seqid=0 2024-11-14T09:43:43,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741878_1054 (size=12301) 2024-11-14T09:43:43,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c621da6b3cf14564ae62ce06f788b1e4 2024-11-14T09:43:43,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/97f8c77027504c75b36b4f49eb9656eb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/97f8c77027504c75b36b4f49eb9656eb 2024-11-14T09:43:43,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577483131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577483131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577483132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577483132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577483132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/97f8c77027504c75b36b4f49eb9656eb, entries=200, sequenceid=323, filesize=14.4 K 2024-11-14T09:43:43,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/46e2ed46f473414dac0e4379370d5c0d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/46e2ed46f473414dac0e4379370d5c0d 2024-11-14T09:43:43,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/46e2ed46f473414dac0e4379370d5c0d, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T09:43:43,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c621da6b3cf14564ae62ce06f788b1e4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c621da6b3cf14564ae62ce06f788b1e4 2024-11-14T09:43:43,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c621da6b3cf14564ae62ce06f788b1e4, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T09:43:43,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 52ef1722f0e126ebd82cd709b2dcdf9b in 214ms, sequenceid=323, compaction requested=true 2024-11-14T09:43:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:43,177 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:43,177 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:43,180 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42159 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:43,180 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:43,180 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:43,180 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/541b677fe22b43278d4646a71a367fe0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/95d164ef01134f76964cb2265cd90e53, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/97f8c77027504c75b36b4f49eb9656eb] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=41.2 K 2024-11-14T09:43:43,182 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541b677fe22b43278d4646a71a367fe0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731577420738 2024-11-14T09:43:43,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:43,183 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d164ef01134f76964cb2265cd90e53, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731577421944 2024-11-14T09:43:43,183 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f8c77027504c75b36b4f49eb9656eb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731577422613 2024-11-14T09:43:43,188 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:43,188 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:43,189 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d8c357222ad5451d88892d44994a41f7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9459d068d2f40f5a3f15cfe7c52903b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/46e2ed46f473414dac0e4379370d5c0d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=36.4 K 2024-11-14T09:43:43,189 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c357222ad5451d88892d44994a41f7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731577420738 2024-11-14T09:43:43,190 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f9459d068d2f40f5a3f15cfe7c52903b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731577422586 2024-11-14T09:43:43,191 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e2ed46f473414dac0e4379370d5c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731577422613 2024-11-14T09:43:43,224 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:43,225 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/48ea1eeb97a640e9ac52200b8e101115 is 50, key is test_row_0/A:col10/1731577422613/Put/seqid=0 2024-11-14T09:43:43,236 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:43,237 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/9ae176060a654fadbb73e80358608508 is 50, key is test_row_0/B:col10/1731577422613/Put/seqid=0 2024-11-14T09:43:43,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741879_1055 (size=12779) 2024-11-14T09:43:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741880_1056 (size=12779) 2024-11-14T09:43:43,271 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/9ae176060a654fadbb73e80358608508 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9ae176060a654fadbb73e80358608508 2024-11-14T09:43:43,271 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/48ea1eeb97a640e9ac52200b8e101115 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/48ea1eeb97a640e9ac52200b8e101115 2024-11-14T09:43:43,283 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into 9ae176060a654fadbb73e80358608508(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:43,284 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:43,284 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=13, startTime=1731577423177; duration=0sec 2024-11-14T09:43:43,284 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:43,284 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:43,284 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:43,286 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 48ea1eeb97a640e9ac52200b8e101115(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:43,286 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:43,286 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=13, startTime=1731577423177; duration=0sec 2024-11-14T09:43:43,286 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:43,286 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:43,289 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:43,289 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:43,289 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:43,289 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/35f85c7ea70b4eb9a9c917b87d6d7921, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/1fdf9957698f42ca887fa471c7a2aecf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c621da6b3cf14564ae62ce06f788b1e4] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=36.4 K 2024-11-14T09:43:43,290 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f85c7ea70b4eb9a9c917b87d6d7921, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731577420738 2024-11-14T09:43:43,291 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fdf9957698f42ca887fa471c7a2aecf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731577422586 2024-11-14T09:43:43,291 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c621da6b3cf14564ae62ce06f788b1e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731577422613 2024-11-14T09:43:43,308 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:43,309 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/40e88b9357f44873a5d1854d20327479 is 50, key is test_row_0/C:col10/1731577422613/Put/seqid=0 2024-11-14T09:43:43,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:43:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:43,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741881_1057 (size=12779) 2024-11-14T09:43:43,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:43,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:43,360 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/40e88b9357f44873a5d1854d20327479 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/40e88b9357f44873a5d1854d20327479 2024-11-14T09:43:43,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/3ea98686d71b4a39a5223d15caccd500 is 50, key is test_row_0/A:col10/1731577423343/Put/seqid=0 2024-11-14T09:43:43,372 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into 40e88b9357f44873a5d1854d20327479(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:43,372 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:43,373 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=13, startTime=1731577423177; duration=0sec 2024-11-14T09:43:43,373 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:43,373 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577483369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577483370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577483373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577483376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577483378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741882_1058 (size=14741) 2024-11-14T09:43:43,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/3ea98686d71b4a39a5223d15caccd500 2024-11-14T09:43:43,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/3f1843a2bc4a47729728d9e776626293 is 50, key is test_row_0/B:col10/1731577423343/Put/seqid=0 2024-11-14T09:43:43,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741883_1059 (size=12301) 2024-11-14T09:43:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577483490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577483491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577483491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577483491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577483500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:43:43,561 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:43,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:43:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T09:43:43,567 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:43:43,571 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:43:43,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:43:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:43:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:43:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577483700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577483700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577483700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577483700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577483711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:43,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:43:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:43,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:43,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/3f1843a2bc4a47729728d9e776626293 2024-11-14T09:43:43,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/5aae49feb2864624909e8dafdcaabb67 is 50, key is test_row_0/C:col10/1731577423343/Put/seqid=0 2024-11-14T09:43:43,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741884_1060 (size=12301) 2024-11-14T09:43:43,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:43:43,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:43,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:43:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577484010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577484010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577484011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577484010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577484020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:43:44,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:44,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:44,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:44,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:44,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:44,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:43:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:44,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:44,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:43:44,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/5aae49feb2864624909e8dafdcaabb67 2024-11-14T09:43:44,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/3ea98686d71b4a39a5223d15caccd500 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/3ea98686d71b4a39a5223d15caccd500 2024-11-14T09:43:44,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/3ea98686d71b4a39a5223d15caccd500, entries=200, sequenceid=342, filesize=14.4 K 2024-11-14T09:43:44,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/3f1843a2bc4a47729728d9e776626293 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/3f1843a2bc4a47729728d9e776626293 2024-11-14T09:43:44,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/3f1843a2bc4a47729728d9e776626293, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:43:44,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/5aae49feb2864624909e8dafdcaabb67 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5aae49feb2864624909e8dafdcaabb67 2024-11-14T09:43:44,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5aae49feb2864624909e8dafdcaabb67, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:43:44,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 52ef1722f0e126ebd82cd709b2dcdf9b in 951ms, sequenceid=342, compaction requested=false 2024-11-14T09:43:44,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:44,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:43:44,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:44,358 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:43:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:44,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b4883e6f1b7e46ccb5a579e809f18507 is 50, key is test_row_0/A:col10/1731577423366/Put/seqid=0 2024-11-14T09:43:44,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741885_1061 (size=12301) 2024-11-14T09:43:44,432 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b4883e6f1b7e46ccb5a579e809f18507 2024-11-14T09:43:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/d4f62a31e7994f55bf5f785bd87e6acf is 50, key is test_row_0/B:col10/1731577423366/Put/seqid=0 2024-11-14T09:43:44,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741886_1062 (size=12301) 2024-11-14T09:43:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:44,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577484553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577484556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577484556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577484564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577484572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577484670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577484671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577484671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577484681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577484681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:43:44,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577484880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577484880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577484880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577484891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577484901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:44,917 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/d4f62a31e7994f55bf5f785bd87e6acf 2024-11-14T09:43:44,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/d25d3ecf6b0f42a7a9717260efb8e358 is 50, key is test_row_0/C:col10/1731577423366/Put/seqid=0 2024-11-14T09:43:44,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741887_1063 (size=12301) 2024-11-14T09:43:44,948 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/d25d3ecf6b0f42a7a9717260efb8e358 2024-11-14T09:43:44,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b4883e6f1b7e46ccb5a579e809f18507 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b4883e6f1b7e46ccb5a579e809f18507 2024-11-14T09:43:44,963 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b4883e6f1b7e46ccb5a579e809f18507, entries=150, sequenceid=358, filesize=12.0 K 2024-11-14T09:43:44,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/d4f62a31e7994f55bf5f785bd87e6acf as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d4f62a31e7994f55bf5f785bd87e6acf 2024-11-14T09:43:44,973 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d4f62a31e7994f55bf5f785bd87e6acf, entries=150, sequenceid=358, filesize=12.0 K 2024-11-14T09:43:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/d25d3ecf6b0f42a7a9717260efb8e358 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/d25d3ecf6b0f42a7a9717260efb8e358 2024-11-14T09:43:44,980 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/d25d3ecf6b0f42a7a9717260efb8e358, entries=150, sequenceid=358, filesize=12.0 K 2024-11-14T09:43:44,982 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 52ef1722f0e126ebd82cd709b2dcdf9b in 625ms, sequenceid=358, compaction requested=true 2024-11-14T09:43:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T09:43:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T09:43:44,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T09:43:44,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-14T09:43:44,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4330 sec 2024-11-14T09:43:45,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T09:43:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:45,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/383df07bd7734556aa82a4871214bc22 is 50, key is test_row_0/A:col10/1731577424555/Put/seqid=0 2024-11-14T09:43:45,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577485246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577485252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577485252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741888_1064 (size=14741) 2024-11-14T09:43:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577485254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577485254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577485372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577485381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577485382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577485382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577485382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577485580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577485591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577485600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577485600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577485600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/383df07bd7734556aa82a4871214bc22 2024-11-14T09:43:45,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ca0074bb1bdf47f8a963ccdb49b7f007 is 50, key is test_row_0/B:col10/1731577424555/Put/seqid=0 2024-11-14T09:43:45,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741889_1065 (size=12301) 2024-11-14T09:43:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:43:45,723 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:45,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:43:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T09:43:45,729 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:43:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:43:45,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:43:45,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:43:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:43:45,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:43:45,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:45,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:45,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:45,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:45,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577485902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577485903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577485911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577485911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577485912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:46,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:43:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:46,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:43:46,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ca0074bb1bdf47f8a963ccdb49b7f007 2024-11-14T09:43:46,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/534ddf1d3d63464993161c0a2a4dd763 is 50, key is test_row_0/C:col10/1731577424555/Put/seqid=0 2024-11-14T09:43:46,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741890_1066 (size=12301) 2024-11-14T09:43:46,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:43:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:43:46,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:46,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:43:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577486420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577486420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:46,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577486421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:46,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577486421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577486430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:46,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:43:46,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:46,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:46,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/534ddf1d3d63464993161c0a2a4dd763 2024-11-14T09:43:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/383df07bd7734556aa82a4871214bc22 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/383df07bd7734556aa82a4871214bc22 2024-11-14T09:43:46,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/383df07bd7734556aa82a4871214bc22, entries=200, sequenceid=376, filesize=14.4 K 2024-11-14T09:43:46,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/ca0074bb1bdf47f8a963ccdb49b7f007 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ca0074bb1bdf47f8a963ccdb49b7f007 2024-11-14T09:43:46,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ca0074bb1bdf47f8a963ccdb49b7f007, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T09:43:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/534ddf1d3d63464993161c0a2a4dd763 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/534ddf1d3d63464993161c0a2a4dd763 2024-11-14T09:43:46,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/534ddf1d3d63464993161c0a2a4dd763, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T09:43:46,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1380ms, sequenceid=376, compaction requested=true 2024-11-14T09:43:46,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:46,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:46,574 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:46,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:46,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:46,574 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:46,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:46,576 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:46,576 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54562 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:46,576 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:46,576 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:46,576 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,576 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,576 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/48ea1eeb97a640e9ac52200b8e101115, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/3ea98686d71b4a39a5223d15caccd500, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b4883e6f1b7e46ccb5a579e809f18507, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/383df07bd7734556aa82a4871214bc22] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=53.3 K 2024-11-14T09:43:46,576 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9ae176060a654fadbb73e80358608508, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/3f1843a2bc4a47729728d9e776626293, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d4f62a31e7994f55bf5f785bd87e6acf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ca0074bb1bdf47f8a963ccdb49b7f007] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=48.5 K 2024-11-14T09:43:46,577 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae176060a654fadbb73e80358608508, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731577422613 2024-11-14T09:43:46,577 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48ea1eeb97a640e9ac52200b8e101115, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731577422613 2024-11-14T09:43:46,577 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f1843a2bc4a47729728d9e776626293, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577423007 2024-11-14T09:43:46,578 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ea98686d71b4a39a5223d15caccd500, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577423007 2024-11-14T09:43:46,578 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f62a31e7994f55bf5f785bd87e6acf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731577423366 2024-11-14T09:43:46,581 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0074bb1bdf47f8a963ccdb49b7f007, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731577424552 2024-11-14T09:43:46,581 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4883e6f1b7e46ccb5a579e809f18507, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731577423366 2024-11-14T09:43:46,582 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 383df07bd7734556aa82a4871214bc22, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731577424541 2024-11-14T09:43:46,607 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:46,607 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:46,608 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8426071272b247229e7084e00b982fd9 is 50, key is test_row_0/A:col10/1731577424555/Put/seqid=0 2024-11-14T09:43:46,608 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4607c180216b413e93ca997255be1417 is 50, key is test_row_0/B:col10/1731577424555/Put/seqid=0 2024-11-14T09:43:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741891_1067 (size=12915) 2024-11-14T09:43:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741892_1068 (size=12915) 2024-11-14T09:43:46,651 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8426071272b247229e7084e00b982fd9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8426071272b247229e7084e00b982fd9 2024-11-14T09:43:46,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:43:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,665 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:43:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:46,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:46,671 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4607c180216b413e93ca997255be1417 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4607c180216b413e93ca997255be1417 2024-11-14T09:43:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/0e61fae381b846398a3bffbef1b00198 is 50, key is test_row_0/A:col10/1731577425245/Put/seqid=0 2024-11-14T09:43:46,676 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 8426071272b247229e7084e00b982fd9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:46,676 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:46,676 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=12, startTime=1731577426574; duration=0sec 2024-11-14T09:43:46,676 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:46,676 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:46,676 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:46,685 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into 4607c180216b413e93ca997255be1417(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:46,685 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:46,685 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=12, startTime=1731577426574; duration=0sec 2024-11-14T09:43:46,685 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:46,686 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:46,692 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:46,693 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:46,693 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,693 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/40e88b9357f44873a5d1854d20327479, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5aae49feb2864624909e8dafdcaabb67, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/d25d3ecf6b0f42a7a9717260efb8e358, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/534ddf1d3d63464993161c0a2a4dd763] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=48.5 K 2024-11-14T09:43:46,694 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e88b9357f44873a5d1854d20327479, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731577422613 2024-11-14T09:43:46,695 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aae49feb2864624909e8dafdcaabb67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577423007 2024-11-14T09:43:46,696 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d25d3ecf6b0f42a7a9717260efb8e358, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731577423366 2024-11-14T09:43:46,697 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 534ddf1d3d63464993161c0a2a4dd763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731577424552 2024-11-14T09:43:46,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741893_1069 (size=12301) 2024-11-14T09:43:46,735 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/0e61fae381b846398a3bffbef1b00198 2024-11-14T09:43:46,746 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:46,746 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/789a1efe7d244674a6b6d5fd0eb508e6 is 50, key is test_row_0/C:col10/1731577424555/Put/seqid=0 2024-11-14T09:43:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/12183cea449b42c0ab97b1bc0efd5a5c is 50, key is test_row_0/B:col10/1731577425245/Put/seqid=0 2024-11-14T09:43:46,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741894_1070 (size=12915) 2024-11-14T09:43:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741895_1071 (size=12301) 2024-11-14T09:43:46,806 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/12183cea449b42c0ab97b1bc0efd5a5c 2024-11-14T09:43:46,807 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/789a1efe7d244674a6b6d5fd0eb508e6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/789a1efe7d244674a6b6d5fd0eb508e6 2024-11-14T09:43:46,828 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into 789a1efe7d244674a6b6d5fd0eb508e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:46,828 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:46,828 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=12, startTime=1731577426574; duration=0sec 2024-11-14T09:43:46,829 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:46,829 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/259c4ece0f76408990b4d5a75959a9d0 is 50, key is test_row_0/C:col10/1731577425245/Put/seqid=0 2024-11-14T09:43:46,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741896_1072 (size=12301) 2024-11-14T09:43:46,857 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/259c4ece0f76408990b4d5a75959a9d0 2024-11-14T09:43:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:43:46,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/0e61fae381b846398a3bffbef1b00198 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0e61fae381b846398a3bffbef1b00198 2024-11-14T09:43:46,881 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0e61fae381b846398a3bffbef1b00198, entries=150, sequenceid=391, filesize=12.0 K 2024-11-14T09:43:46,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/12183cea449b42c0ab97b1bc0efd5a5c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/12183cea449b42c0ab97b1bc0efd5a5c 2024-11-14T09:43:46,901 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/12183cea449b42c0ab97b1bc0efd5a5c, entries=150, sequenceid=391, filesize=12.0 K 2024-11-14T09:43:46,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/259c4ece0f76408990b4d5a75959a9d0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/259c4ece0f76408990b4d5a75959a9d0 2024-11-14T09:43:46,911 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/259c4ece0f76408990b4d5a75959a9d0, entries=150, sequenceid=391, filesize=12.0 K 2024-11-14T09:43:46,914 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 52ef1722f0e126ebd82cd709b2dcdf9b in 248ms, sequenceid=391, compaction requested=false 2024-11-14T09:43:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T09:43:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T09:43:46,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T09:43:46,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1900 sec 2024-11-14T09:43:46,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2080 sec 2024-11-14T09:43:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:47,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:43:47,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:47,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:47,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:47,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:47,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:47,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:47,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/23ba956de66b416986de865bc337f7bd is 50, key is test_row_0/A:col10/1731577427443/Put/seqid=0 2024-11-14T09:43:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741897_1073 (size=17181) 2024-11-14T09:43:47,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/23ba956de66b416986de865bc337f7bd 2024-11-14T09:43:47,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4a6f89fcc37741148390968d96e621fe is 50, key is test_row_0/B:col10/1731577427443/Put/seqid=0 2024-11-14T09:43:47,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741898_1074 (size=12301) 2024-11-14T09:43:47,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4a6f89fcc37741148390968d96e621fe 2024-11-14T09:43:47,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577427543, max timestamp=1731577427568Num uniques 859; 2024-11-14T09:43:47,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577427543, max timestamp=1731577427568Num uniques 859; 2024-11-14T09:43:47,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:43:47,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:43:47,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577427543, max timestamp=1731577427568Num uniques 859; 2024-11-14T09:43:47,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:43:47,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/14a4a9511d76469fb63086ba550c21c1 is 50, key is test_row_0/C:col10/1731577427443/Put/seqid=0 2024-11-14T09:43:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577487621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577487623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577487623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577487626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577487629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741899_1075 (size=12301) 2024-11-14T09:43:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/14a4a9511d76469fb63086ba550c21c1 2024-11-14T09:43:47,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/23ba956de66b416986de865bc337f7bd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/23ba956de66b416986de865bc337f7bd 2024-11-14T09:43:47,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/23ba956de66b416986de865bc337f7bd, entries=250, sequenceid=404, filesize=16.8 K 2024-11-14T09:43:47,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/4a6f89fcc37741148390968d96e621fe as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4a6f89fcc37741148390968d96e621fe 2024-11-14T09:43:47,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4a6f89fcc37741148390968d96e621fe, entries=150, sequenceid=404, filesize=12.0 K 2024-11-14T09:43:47,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/14a4a9511d76469fb63086ba550c21c1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/14a4a9511d76469fb63086ba550c21c1 2024-11-14T09:43:47,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/14a4a9511d76469fb63086ba550c21c1, entries=150, sequenceid=404, filesize=12.0 K 2024-11-14T09:43:47,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 52ef1722f0e126ebd82cd709b2dcdf9b in 239ms, sequenceid=404, compaction requested=true 2024-11-14T09:43:47,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:47,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:47,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:47,703 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:47,703 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:47,704 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:47,705 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:47,705 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:47,705 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4607c180216b413e93ca997255be1417, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/12183cea449b42c0ab97b1bc0efd5a5c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4a6f89fcc37741148390968d96e621fe] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=36.6 K 2024-11-14T09:43:47,706 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4607c180216b413e93ca997255be1417, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731577424552 2024-11-14T09:43:47,707 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:47,707 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:47,707 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:47,707 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8426071272b247229e7084e00b982fd9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0e61fae381b846398a3bffbef1b00198, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/23ba956de66b416986de865bc337f7bd] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=41.4 K 2024-11-14T09:43:47,707 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 12183cea449b42c0ab97b1bc0efd5a5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731577425225 2024-11-14T09:43:47,708 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8426071272b247229e7084e00b982fd9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731577424552 2024-11-14T09:43:47,708 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a6f89fcc37741148390968d96e621fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731577427443 2024-11-14T09:43:47,708 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e61fae381b846398a3bffbef1b00198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731577425225 2024-11-14T09:43:47,709 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ba956de66b416986de865bc337f7bd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731577427441 2024-11-14T09:43:47,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:47,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:47,732 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:47,733 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e87bc620191c4a879d0b469897e210f6 is 50, key is test_row_0/B:col10/1731577427443/Put/seqid=0 2024-11-14T09:43:47,751 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:47,751 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/0b0ea6b84ab54f65b033ce4931f8bb83 is 50, key is test_row_0/A:col10/1731577427443/Put/seqid=0 2024-11-14T09:43:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:47,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-14T09:43:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:47,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/2976cf81840049198ca05ff4ae8a4b7d is 50, key is test_row_0/A:col10/1731577427619/Put/seqid=0 2024-11-14T09:43:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577487768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741900_1076 (size=13017) 2024-11-14T09:43:47,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577487776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577487778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577487777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577487779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,803 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/e87bc620191c4a879d0b469897e210f6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e87bc620191c4a879d0b469897e210f6 2024-11-14T09:43:47,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741901_1077 (size=13017) 2024-11-14T09:43:47,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741902_1078 (size=17181) 2024-11-14T09:43:47,821 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/0b0ea6b84ab54f65b033ce4931f8bb83 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0b0ea6b84ab54f65b033ce4931f8bb83 2024-11-14T09:43:47,823 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into e87bc620191c4a879d0b469897e210f6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:47,823 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:47,823 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=13, startTime=1731577427703; duration=0sec 2024-11-14T09:43:47,823 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:47,823 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:47,823 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:47,826 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:47,828 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:47,832 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 0b0ea6b84ab54f65b033ce4931f8bb83(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:47,832 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:47,832 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=13, startTime=1731577427702; duration=0sec 2024-11-14T09:43:47,832 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:47,832 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:47,832 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:47,832 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/789a1efe7d244674a6b6d5fd0eb508e6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/259c4ece0f76408990b4d5a75959a9d0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/14a4a9511d76469fb63086ba550c21c1] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=36.6 K 2024-11-14T09:43:47,833 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 789a1efe7d244674a6b6d5fd0eb508e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731577424552 2024-11-14T09:43:47,833 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 259c4ece0f76408990b4d5a75959a9d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731577425225 2024-11-14T09:43:47,834 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a4a9511d76469fb63086ba550c21c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731577427443 2024-11-14T09:43:47,878 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:47,879 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b55f4742d4084194a6bd75140b17acdf is 50, key is test_row_0/C:col10/1731577427443/Put/seqid=0 2024-11-14T09:43:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:43:47,881 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:47,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:43:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T09:43:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:43:47,887 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:43:47,888 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:43:47,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:43:47,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577487890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577487901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577487901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577487901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577487901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:47,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741903_1079 (size=13017) 2024-11-14T09:43:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:43:48,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577488101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577488120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577488120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577488120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577488120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:48,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:48,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:43:48,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/2976cf81840049198ca05ff4ae8a4b7d 2024-11-14T09:43:48,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/cec276104af340779a40c7ee9d3b18f8 is 50, key is test_row_0/B:col10/1731577427619/Put/seqid=0 2024-11-14T09:43:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741904_1080 (size=12301) 2024-11-14T09:43:48,331 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b55f4742d4084194a6bd75140b17acdf as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b55f4742d4084194a6bd75140b17acdf 2024-11-14T09:43:48,340 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into b55f4742d4084194a6bd75140b17acdf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:48,340 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:48,340 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=13, startTime=1731577427711; duration=0sec 2024-11-14T09:43:48,340 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:48,341 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:48,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:48,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:48,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577488410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577488430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577488430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577488430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577488430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:48,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:48,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:43:48,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/cec276104af340779a40c7ee9d3b18f8 2024-11-14T09:43:48,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/33ade06324534ba3902f12be2765968f is 50, key is test_row_0/C:col10/1731577427619/Put/seqid=0 2024-11-14T09:43:48,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741905_1081 (size=12301) 2024-11-14T09:43:48,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577488921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577488940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577488940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577488940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577488951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:48,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:48,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:48,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:43:49,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/33ade06324534ba3902f12be2765968f 2024-11-14T09:43:49,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/2976cf81840049198ca05ff4ae8a4b7d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/2976cf81840049198ca05ff4ae8a4b7d 2024-11-14T09:43:49,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/2976cf81840049198ca05ff4ae8a4b7d, entries=250, sequenceid=441, filesize=16.8 K 2024-11-14T09:43:49,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/cec276104af340779a40c7ee9d3b18f8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/cec276104af340779a40c7ee9d3b18f8 2024-11-14T09:43:49,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/cec276104af340779a40c7ee9d3b18f8, entries=150, sequenceid=441, filesize=12.0 K 2024-11-14T09:43:49,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/33ade06324534ba3902f12be2765968f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/33ade06324534ba3902f12be2765968f 2024-11-14T09:43:49,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/33ade06324534ba3902f12be2765968f, entries=150, sequenceid=441, filesize=12.0 K 2024-11-14T09:43:49,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1340ms, sequenceid=441, compaction requested=false 2024-11-14T09:43:49,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:49,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:43:49,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:49,133 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T09:43:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:49,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/30a22ea6d55040e1acfe5e836f364d4e is 50, key is test_row_0/A:col10/1731577427775/Put/seqid=0 2024-11-14T09:43:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741906_1082 (size=9857) 2024-11-14T09:43:49,558 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/30a22ea6d55040e1acfe5e836f364d4e 2024-11-14T09:43:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/fd1ad96ba4054e2ab189b69efb76433d is 50, key is test_row_0/B:col10/1731577427775/Put/seqid=0 2024-11-14T09:43:49,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741907_1083 (size=9857) 2024-11-14T09:43:49,595 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/fd1ad96ba4054e2ab189b69efb76433d 2024-11-14T09:43:49,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/ad9357f252634b13b976177e3f2a5934 is 50, key is test_row_0/C:col10/1731577427775/Put/seqid=0 2024-11-14T09:43:49,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741908_1084 (size=9857) 2024-11-14T09:43:49,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:49,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577429971, max timestamp=1731577429994Num uniques 859; 2024-11-14T09:43:49,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:43:50,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577429971, max timestamp=1731577429994Num uniques 859; 2024-11-14T09:43:50,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:43:50,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731577429971, max timestamp=1731577429994Num uniques 859; 2024-11-14T09:43:50,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-14T09:43:50,028 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/ad9357f252634b13b976177e3f2a5934 2024-11-14T09:43:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:43:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/30a22ea6d55040e1acfe5e836f364d4e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/30a22ea6d55040e1acfe5e836f364d4e 2024-11-14T09:43:50,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577490027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577490028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577490031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731577430017, max timestamp=1731577430033Num uniques 1068; 2024-11-14T09:43:50,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T09:43:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731577430017, max timestamp=1731577430033Num uniques 1068; 2024-11-14T09:43:50,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-14T09:43:50,049 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/30a22ea6d55040e1acfe5e836f364d4e, entries=100, sequenceid=453, filesize=9.6 K 2024-11-14T09:43:50,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731577430017, max timestamp=1731577430033Num uniques 1009; 2024-11-14T09:43:50,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-14T09:43:50,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/fd1ad96ba4054e2ab189b69efb76433d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/fd1ad96ba4054e2ab189b69efb76433d 2024-11-14T09:43:50,068 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/fd1ad96ba4054e2ab189b69efb76433d, entries=100, sequenceid=453, filesize=9.6 K 2024-11-14T09:43:50,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/ad9357f252634b13b976177e3f2a5934 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ad9357f252634b13b976177e3f2a5934 2024-11-14T09:43:50,090 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ad9357f252634b13b976177e3f2a5934, entries=100, sequenceid=453, filesize=9.6 K 2024-11-14T09:43:50,091 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=61.02 KB/62484 for 52ef1722f0e126ebd82cd709b2dcdf9b in 959ms, sequenceid=453, compaction requested=true 2024-11-14T09:43:50,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:50,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:50,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T09:43:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T09:43:50,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=67.73 KB heapSize=186.34 KB 2024-11-14T09:43:50,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:50,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:50,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:50,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:50,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:50,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:50,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T09:43:50,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2070 sec 2024-11-14T09:43:50,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2220 sec 2024-11-14T09:43:50,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/a8648044903740f28e1206614d4cff76 is 50, key is test_row_0/A:col10/1731577430069/Put/seqid=0 2024-11-14T09:43:50,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741909_1085 (size=14741) 2024-11-14T09:43:50,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/a8648044903740f28e1206614d4cff76 2024-11-14T09:43:50,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/9e93c3bf88744d00928f06099223b31c is 50, key is test_row_0/B:col10/1731577430069/Put/seqid=0 2024-11-14T09:43:50,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577490175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577490177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577490179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577490179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577490181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741910_1086 (size=12301) 2024-11-14T09:43:50,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577490290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577490300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577490301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577490301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577490310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577490501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577490512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577490520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577490522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577490521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:50,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.56 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/9e93c3bf88744d00928f06099223b31c 2024-11-14T09:43:50,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/090ccd7e7e65487aa4056cc4dcc22252 is 50, key is test_row_0/C:col10/1731577430069/Put/seqid=0 2024-11-14T09:43:50,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741911_1087 (size=12301) 2024-11-14T09:43:50,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.56 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/090ccd7e7e65487aa4056cc4dcc22252 2024-11-14T09:43:50,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/a8648044903740f28e1206614d4cff76 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a8648044903740f28e1206614d4cff76 2024-11-14T09:43:50,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a8648044903740f28e1206614d4cff76, entries=200, sequenceid=497, filesize=14.4 K 2024-11-14T09:43:50,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/9e93c3bf88744d00928f06099223b31c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9e93c3bf88744d00928f06099223b31c 2024-11-14T09:43:50,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9e93c3bf88744d00928f06099223b31c, entries=150, sequenceid=497, filesize=12.0 K 2024-11-14T09:43:50,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/090ccd7e7e65487aa4056cc4dcc22252 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/090ccd7e7e65487aa4056cc4dcc22252 2024-11-14T09:43:50,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/090ccd7e7e65487aa4056cc4dcc22252, entries=150, sequenceid=497, filesize=12.0 K 2024-11-14T09:43:50,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.73 KB/69354, heapSize ~186.30 KB/190768, currentSize=114.05 KB/116790 for 52ef1722f0e126ebd82cd709b2dcdf9b in 666ms, sequenceid=497, compaction requested=true 2024-11-14T09:43:50,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:50,759 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:50,759 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:50,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:50,761 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:50,762 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:50,762 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:50,762 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e87bc620191c4a879d0b469897e210f6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/cec276104af340779a40c7ee9d3b18f8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/fd1ad96ba4054e2ab189b69efb76433d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9e93c3bf88744d00928f06099223b31c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=46.4 K 2024-11-14T09:43:50,763 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54796 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:50,763 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:50,763 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:50,763 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0b0ea6b84ab54f65b033ce4931f8bb83, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/2976cf81840049198ca05ff4ae8a4b7d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/30a22ea6d55040e1acfe5e836f364d4e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a8648044903740f28e1206614d4cff76] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=53.5 K 2024-11-14T09:43:50,763 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e87bc620191c4a879d0b469897e210f6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731577427443 2024-11-14T09:43:50,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:50,764 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b0ea6b84ab54f65b033ce4931f8bb83, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731577427443 2024-11-14T09:43:50,764 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2976cf81840049198ca05ff4ae8a4b7d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731577427611 2024-11-14T09:43:50,765 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30a22ea6d55040e1acfe5e836f364d4e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731577427775 2024-11-14T09:43:50,765 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8648044903740f28e1206614d4cff76, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731577430061 2024-11-14T09:43:50,765 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting cec276104af340779a40c7ee9d3b18f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731577427619 2024-11-14T09:43:50,766 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1ad96ba4054e2ab189b69efb76433d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731577427775 2024-11-14T09:43:50,767 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e93c3bf88744d00928f06099223b31c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731577430069 2024-11-14T09:43:50,815 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:50,815 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:50,817 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/36dbc0a38a544398b673017472457bac is 50, key is test_row_0/A:col10/1731577430069/Put/seqid=0 2024-11-14T09:43:50,817 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/21d0f5cc22484e68baf574b72916937b is 50, key is test_row_0/B:col10/1731577430069/Put/seqid=0 2024-11-14T09:43:50,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577430170, max timestamp=1731577430822Num uniques 859; 2024-11-14T09:43:50,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577430170, max timestamp=1731577430822Num uniques 859; 2024-11-14T09:43:50,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:50,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:50,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577430170, max timestamp=1731577430824Num uniques 859; 2024-11-14T09:43:50,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:50,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-14T09:43:50,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:50,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:50,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:50,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:50,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:50,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:50,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741912_1088 (size=13153) 2024-11-14T09:43:50,881 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/36dbc0a38a544398b673017472457bac as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/36dbc0a38a544398b673017472457bac 2024-11-14T09:43:50,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c872ef30159e4bb2a37af63d4d870dfd is 50, key is test_row_0/A:col10/1731577430836/Put/seqid=0 2024-11-14T09:43:50,891 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 36dbc0a38a544398b673017472457bac(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:50,891 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:50,892 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=12, startTime=1731577430758; duration=0sec 2024-11-14T09:43:50,896 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:50,896 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:50,896 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:43:50,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741913_1089 (size=13153) 2024-11-14T09:43:50,901 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:43:50,903 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:50,903 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:50,903 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b55f4742d4084194a6bd75140b17acdf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/33ade06324534ba3902f12be2765968f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ad9357f252634b13b976177e3f2a5934, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/090ccd7e7e65487aa4056cc4dcc22252] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=46.4 K 2024-11-14T09:43:50,904 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b55f4742d4084194a6bd75140b17acdf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731577427443 2024-11-14T09:43:50,905 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ade06324534ba3902f12be2765968f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731577427619 2024-11-14T09:43:50,907 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9357f252634b13b976177e3f2a5934, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731577427775 2024-11-14T09:43:50,908 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 090ccd7e7e65487aa4056cc4dcc22252, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731577430069 2024-11-14T09:43:50,918 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/21d0f5cc22484e68baf574b72916937b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/21d0f5cc22484e68baf574b72916937b 2024-11-14T09:43:50,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741914_1090 (size=19621) 2024-11-14T09:43:50,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c872ef30159e4bb2a37af63d4d870dfd 2024-11-14T09:43:50,949 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into 21d0f5cc22484e68baf574b72916937b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:50,949 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:50,949 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=12, startTime=1731577430759; duration=0sec 2024-11-14T09:43:50,949 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:50,949 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:50,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577430890, max timestamp=1731577430951Num uniques 859; 2024-11-14T09:43:50,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:43:50,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/c4a65fbb0de14fc88632091d2ba2e781 is 50, key is test_row_0/B:col10/1731577430836/Put/seqid=0 2024-11-14T09:43:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577430890, max timestamp=1731577430951Num uniques 859; 2024-11-14T09:43:50,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:43:50,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577430890, max timestamp=1731577430951Num uniques 859; 2024-11-14T09:43:50,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:43:50,974 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#79 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:50,977 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8ef3aadebe0c4995b579512f987a5369 is 50, key is test_row_0/C:col10/1731577430069/Put/seqid=0 2024-11-14T09:43:51,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577491005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577491008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741915_1091 (size=12301) 2024-11-14T09:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577491011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741916_1092 (size=13153) 2024-11-14T09:43:51,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/c4a65fbb0de14fc88632091d2ba2e781 2024-11-14T09:43:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577491012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577491022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c91310cf2ae6493091a857352affb6ed is 50, key is test_row_0/C:col10/1731577430836/Put/seqid=0 2024-11-14T09:43:51,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741917_1093 (size=12301) 2024-11-14T09:43:51,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c91310cf2ae6493091a857352affb6ed 2024-11-14T09:43:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c872ef30159e4bb2a37af63d4d870dfd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c872ef30159e4bb2a37af63d4d870dfd 2024-11-14T09:43:51,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c872ef30159e4bb2a37af63d4d870dfd, entries=300, sequenceid=522, filesize=19.2 K 2024-11-14T09:43:51,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/c4a65fbb0de14fc88632091d2ba2e781 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c4a65fbb0de14fc88632091d2ba2e781 2024-11-14T09:43:51,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c4a65fbb0de14fc88632091d2ba2e781, entries=150, sequenceid=522, filesize=12.0 K 2024-11-14T09:43:51,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c91310cf2ae6493091a857352affb6ed as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c91310cf2ae6493091a857352affb6ed 2024-11-14T09:43:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577491131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577491131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577491131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c91310cf2ae6493091a857352affb6ed, entries=150, sequenceid=522, filesize=12.0 K 2024-11-14T09:43:51,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=140.89 KB/144270 for 52ef1722f0e126ebd82cd709b2dcdf9b in 310ms, sequenceid=522, compaction requested=false 2024-11-14T09:43:51,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577491140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:51,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-14T09:43:51,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:51,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:51,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:51,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:51,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:51,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:51,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c3700a7ca4da425b9309f5e5d8cb0edc is 50, key is test_row_0/A:col10/1731577431174/Put/seqid=0 2024-11-14T09:43:51,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741918_1094 (size=14741) 2024-11-14T09:43:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577491253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577491350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577491360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577491361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577491370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577491380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,440 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8ef3aadebe0c4995b579512f987a5369 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8ef3aadebe0c4995b579512f987a5369 2024-11-14T09:43:51,451 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into 8ef3aadebe0c4995b579512f987a5369(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:51,451 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:51,451 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=12, startTime=1731577430760; duration=0sec 2024-11-14T09:43:51,451 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:51,451 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:51,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577491591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c3700a7ca4da425b9309f5e5d8cb0edc 2024-11-14T09:43:51,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/c43dc75170964959b0033e403954ec5b is 50, key is test_row_0/B:col10/1731577431174/Put/seqid=0 2024-11-14T09:43:51,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741919_1095 (size=12301) 2024-11-14T09:43:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577491660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577491670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577491670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577491690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:51,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577491901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:43:52,060 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:52,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:43:52,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/c43dc75170964959b0033e403954ec5b 2024-11-14T09:43:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T09:43:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:43:52,063 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:43:52,064 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:43:52,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:43:52,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/4daadf7edeb04b62816ba8cfe79c07c5 is 50, key is test_row_0/C:col10/1731577431174/Put/seqid=0 2024-11-14T09:43:52,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741920_1096 (size=12301) 2024-11-14T09:43:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:43:52,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577492170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577492180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577492180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:52,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577492200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:52,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:43:52,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:52,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:52,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:43:52,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:43:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:43:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577492410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:52,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/4daadf7edeb04b62816ba8cfe79c07c5 2024-11-14T09:43:52,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/c3700a7ca4da425b9309f5e5d8cb0edc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c3700a7ca4da425b9309f5e5d8cb0edc 2024-11-14T09:43:52,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c3700a7ca4da425b9309f5e5d8cb0edc, entries=200, sequenceid=559, filesize=14.4 K 2024-11-14T09:43:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/c43dc75170964959b0033e403954ec5b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c43dc75170964959b0033e403954ec5b 2024-11-14T09:43:52,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c43dc75170964959b0033e403954ec5b, entries=150, sequenceid=559, filesize=12.0 K 2024-11-14T09:43:52,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/4daadf7edeb04b62816ba8cfe79c07c5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/4daadf7edeb04b62816ba8cfe79c07c5 2024-11-14T09:43:52,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/4daadf7edeb04b62816ba8cfe79c07c5, entries=150, sequenceid=559, filesize=12.0 K 2024-11-14T09:43:52,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1348ms, sequenceid=559, compaction requested=true 2024-11-14T09:43:52,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:43:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,524 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T09:43:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:52,526 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:52,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:52,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:52,527 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:52,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:52,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:52,528 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:52,528 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:52,529 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,529 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/21d0f5cc22484e68baf574b72916937b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c4a65fbb0de14fc88632091d2ba2e781, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c43dc75170964959b0033e403954ec5b] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=36.9 K 2024-11-14T09:43:52,529 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:52,529 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:52,529 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,530 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/36dbc0a38a544398b673017472457bac, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c872ef30159e4bb2a37af63d4d870dfd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c3700a7ca4da425b9309f5e5d8cb0edc] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=46.4 K 2024-11-14T09:43:52,530 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d0f5cc22484e68baf574b72916937b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731577430069 2024-11-14T09:43:52,530 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36dbc0a38a544398b673017472457bac, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731577430069 2024-11-14T09:43:52,530 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a65fbb0de14fc88632091d2ba2e781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731577430822 2024-11-14T09:43:52,530 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c872ef30159e4bb2a37af63d4d870dfd, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731577430172 2024-11-14T09:43:52,531 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c43dc75170964959b0033e403954ec5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731577431009 2024-11-14T09:43:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:52,531 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3700a7ca4da425b9309f5e5d8cb0edc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731577431003 2024-11-14T09:43:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8a38b87643d14f17ae8b40ae4b26ff94 is 50, key is test_row_0/A:col10/1731577431241/Put/seqid=0 2024-11-14T09:43:52,543 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:52,544 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/41542b17d44540f781606b88c5db3980 is 50, key is test_row_0/A:col10/1731577431174/Put/seqid=0 2024-11-14T09:43:52,547 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:52,547 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/6a9960b50bed400787bb5d9f9b0f0526 is 50, key is test_row_0/B:col10/1731577431174/Put/seqid=0 2024-11-14T09:43:52,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741922_1098 (size=13255) 2024-11-14T09:43:52,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741921_1097 (size=13255) 2024-11-14T09:43:52,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741923_1099 (size=9857) 2024-11-14T09:43:52,586 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/41542b17d44540f781606b88c5db3980 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/41542b17d44540f781606b88c5db3980 2024-11-14T09:43:52,594 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 41542b17d44540f781606b88c5db3980(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:52,594 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:52,594 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=13, startTime=1731577432526; duration=0sec 2024-11-14T09:43:52,595 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:52,597 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:52,597 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:52,598 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:52,598 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:52,598 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:52,599 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8ef3aadebe0c4995b579512f987a5369, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c91310cf2ae6493091a857352affb6ed, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/4daadf7edeb04b62816ba8cfe79c07c5] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=36.9 K 2024-11-14T09:43:52,599 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef3aadebe0c4995b579512f987a5369, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731577430069 2024-11-14T09:43:52,599 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c91310cf2ae6493091a857352affb6ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731577430822 2024-11-14T09:43:52,600 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4daadf7edeb04b62816ba8cfe79c07c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731577431009 2024-11-14T09:43:52,617 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:52,618 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/48d4f005db5b4f2aabf6c8d28578f3d9 is 50, key is test_row_0/C:col10/1731577431174/Put/seqid=0 2024-11-14T09:43:52,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741924_1100 (size=13255) 2024-11-14T09:43:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:43:52,982 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8a38b87643d14f17ae8b40ae4b26ff94 2024-11-14T09:43:52,987 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/6a9960b50bed400787bb5d9f9b0f0526 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6a9960b50bed400787bb5d9f9b0f0526 2024-11-14T09:43:52,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/6dea3ccaec1f48d4adb82e80567b1ed0 is 50, key is test_row_0/B:col10/1731577431241/Put/seqid=0 2024-11-14T09:43:53,001 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into 6a9960b50bed400787bb5d9f9b0f0526(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:53,001 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:53,001 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=13, startTime=1731577432527; duration=0sec 2024-11-14T09:43:53,002 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:53,002 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:53,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741925_1101 (size=9857) 2024-11-14T09:43:53,031 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/6dea3ccaec1f48d4adb82e80567b1ed0 2024-11-14T09:43:53,041 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/48d4f005db5b4f2aabf6c8d28578f3d9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/48d4f005db5b4f2aabf6c8d28578f3d9 2024-11-14T09:43:53,054 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into 48d4f005db5b4f2aabf6c8d28578f3d9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:53,054 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:53,054 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=13, startTime=1731577432527; duration=0sec 2024-11-14T09:43:53,054 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:53,054 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/07bd75d7ea5c4459820b2c522001a54e is 50, key is test_row_0/C:col10/1731577431241/Put/seqid=0 2024-11-14T09:43:53,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741926_1102 (size=9857) 2024-11-14T09:43:53,129 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/07bd75d7ea5c4459820b2c522001a54e 2024-11-14T09:43:53,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8a38b87643d14f17ae8b40ae4b26ff94 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8a38b87643d14f17ae8b40ae4b26ff94 2024-11-14T09:43:53,147 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8a38b87643d14f17ae8b40ae4b26ff94, entries=100, sequenceid=569, filesize=9.6 K 2024-11-14T09:43:53,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/6dea3ccaec1f48d4adb82e80567b1ed0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6dea3ccaec1f48d4adb82e80567b1ed0 2024-11-14T09:43:53,161 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6dea3ccaec1f48d4adb82e80567b1ed0, entries=100, sequenceid=569, filesize=9.6 K 2024-11-14T09:43:53,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/07bd75d7ea5c4459820b2c522001a54e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/07bd75d7ea5c4459820b2c522001a54e 2024-11-14T09:43:53,184 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/07bd75d7ea5c4459820b2c522001a54e, entries=100, sequenceid=569, filesize=9.6 K 2024-11-14T09:43:53,186 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 52ef1722f0e126ebd82cd709b2dcdf9b in 661ms, sequenceid=569, compaction requested=false 2024-11-14T09:43:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:53,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T09:43:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T09:43:53,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T09:43:53,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1240 sec 2024-11-14T09:43:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:43:53,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1370 sec 2024-11-14T09:43:53,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:43:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:53,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8c1f5fd475384479a4f912809c89c181 is 50, key is test_row_0/A:col10/1731577433207/Put/seqid=0 2024-11-14T09:43:53,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741927_1103 (size=12301) 2024-11-14T09:43:53,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8c1f5fd475384479a4f912809c89c181 2024-11-14T09:43:53,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/08ec8b687e014f27b102b10d9885eb5e is 50, key is test_row_0/B:col10/1731577433207/Put/seqid=0 2024-11-14T09:43:53,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731577433293, max timestamp=1731577433312Num uniques 859; 2024-11-14T09:43:53,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:43:53,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577433297, max timestamp=1731577433317Num uniques 859; 2024-11-14T09:43:53,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:53,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577433297, max timestamp=1731577433312Num uniques 859; 2024-11-14T09:43:53,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:43:53,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741928_1104 (size=12301) 2024-11-14T09:43:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577493356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577493362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577493363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577493364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48162 deadline: 1731577493421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,442 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 52ef1722f0e126ebd82cd709b2dcdf9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:43:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577493471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577493480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577493481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577493481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577493681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577493690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577493700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577493701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:53,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:53,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/08ec8b687e014f27b102b10d9885eb5e 2024-11-14T09:43:53,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8af8daabebc6427993d96a7578d5c500 is 50, key is test_row_0/C:col10/1731577433207/Put/seqid=0 2024-11-14T09:43:53,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741929_1105 (size=12301) 2024-11-14T09:43:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577493990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577494000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577494010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577494016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8af8daabebc6427993d96a7578d5c500 2024-11-14T09:43:54,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/8c1f5fd475384479a4f912809c89c181 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8c1f5fd475384479a4f912809c89c181 2024-11-14T09:43:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:43:54,211 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:54,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:43:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T09:43:54,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8c1f5fd475384479a4f912809c89c181, entries=150, sequenceid=584, filesize=12.0 K 2024-11-14T09:43:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:43:54,214 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:43:54,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/08ec8b687e014f27b102b10d9885eb5e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/08ec8b687e014f27b102b10d9885eb5e 2024-11-14T09:43:54,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:43:54,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:43:54,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/08ec8b687e014f27b102b10d9885eb5e, entries=150, sequenceid=584, filesize=12.0 K 2024-11-14T09:43:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/8af8daabebc6427993d96a7578d5c500 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8af8daabebc6427993d96a7578d5c500 2024-11-14T09:43:54,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8af8daabebc6427993d96a7578d5c500, entries=150, sequenceid=584, filesize=12.0 K 2024-11-14T09:43:54,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1011ms, sequenceid=584, compaction requested=true 2024-11-14T09:43:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:54,236 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:54,236 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:54,238 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:54,238 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:54,238 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:54,238 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6a9960b50bed400787bb5d9f9b0f0526, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6dea3ccaec1f48d4adb82e80567b1ed0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/08ec8b687e014f27b102b10d9885eb5e] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=34.6 K 2024-11-14T09:43:54,239 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a9960b50bed400787bb5d9f9b0f0526, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731577431009 2024-11-14T09:43:54,239 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:54,239 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:54,239 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:54,239 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/41542b17d44540f781606b88c5db3980, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8a38b87643d14f17ae8b40ae4b26ff94, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8c1f5fd475384479a4f912809c89c181] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=34.6 K 2024-11-14T09:43:54,239 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dea3ccaec1f48d4adb82e80567b1ed0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731577431238 2024-11-14T09:43:54,239 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41542b17d44540f781606b88c5db3980, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731577431009 2024-11-14T09:43:54,240 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ec8b687e014f27b102b10d9885eb5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731577433207 2024-11-14T09:43:54,240 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a38b87643d14f17ae8b40ae4b26ff94, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731577431238 2024-11-14T09:43:54,241 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1f5fd475384479a4f912809c89c181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731577433207 2024-11-14T09:43:54,262 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:54,263 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/26d1654147b9489a9fbf260d3ec38bcb is 50, key is test_row_0/A:col10/1731577433207/Put/seqid=0 2024-11-14T09:43:54,272 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:54,272 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/03e2ff6cf03743309a5179156727bb66 is 50, key is test_row_0/B:col10/1731577433207/Put/seqid=0 2024-11-14T09:43:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741930_1106 (size=13357) 2024-11-14T09:43:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:43:54,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741931_1107 (size=13357) 2024-11-14T09:43:54,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:43:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:54,370 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=118.53 KB heapSize=335.38 KB 2024-11-14T09:43:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:54,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:54,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:43:54,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/a388ad97475f4bf5bb334a0c7b40d665 is 50, key is test_row_0/A:col10/1731577433360/Put/seqid=0 2024-11-14T09:43:54,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741932_1108 (size=12301) 2024-11-14T09:43:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:54,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. as already flushing 2024-11-14T09:43:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:43:54,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577494541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48120 deadline: 1731577494540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577494546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577494550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577494661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577494661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577494661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,724 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/26d1654147b9489a9fbf260d3ec38bcb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/26d1654147b9489a9fbf260d3ec38bcb 2024-11-14T09:43:54,732 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/03e2ff6cf03743309a5179156727bb66 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/03e2ff6cf03743309a5179156727bb66 2024-11-14T09:43:54,733 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into 26d1654147b9489a9fbf260d3ec38bcb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:54,733 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:54,733 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=13, startTime=1731577434236; duration=0sec 2024-11-14T09:43:54,733 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:54,733 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:54,733 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:54,736 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:54,736 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:54,736 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:54,737 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/48d4f005db5b4f2aabf6c8d28578f3d9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/07bd75d7ea5c4459820b2c522001a54e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8af8daabebc6427993d96a7578d5c500] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=34.6 K 2024-11-14T09:43:54,737 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48d4f005db5b4f2aabf6c8d28578f3d9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731577431009 2024-11-14T09:43:54,737 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07bd75d7ea5c4459820b2c522001a54e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731577431238 2024-11-14T09:43:54,738 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8af8daabebc6427993d96a7578d5c500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731577433207 2024-11-14T09:43:54,740 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into 03e2ff6cf03743309a5179156727bb66(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:54,740 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:54,740 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=13, startTime=1731577434236; duration=0sec 2024-11-14T09:43:54,740 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:54,740 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:54,760 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#96 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:54,760 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/9503c1904f804a37a3a94b38a0f2f010 is 50, key is test_row_0/C:col10/1731577433207/Put/seqid=0 2024-11-14T09:43:54,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741933_1109 (size=13357) 2024-11-14T09:43:54,803 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/9503c1904f804a37a3a94b38a0f2f010 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9503c1904f804a37a3a94b38a0f2f010 2024-11-14T09:43:54,817 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into 9503c1904f804a37a3a94b38a0f2f010(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:54,817 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:54,817 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=13, startTime=1731577434236; duration=0sec 2024-11-14T09:43:54,818 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:54,818 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:43:54,856 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/a388ad97475f4bf5bb334a0c7b40d665 2024-11-14T09:43:54,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48110 deadline: 1731577494870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731577494880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48146 deadline: 1731577494880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:43:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:43:54,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/10706bcb5432403b9afe3c1c1a977112 is 50, key is test_row_0/B:col10/1731577433360/Put/seqid=0 2024-11-14T09:43:54,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741934_1110 (size=12301) 2024-11-14T09:43:54,921 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/10706bcb5432403b9afe3c1c1a977112 2024-11-14T09:43:54,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/3b10d1882a2d49b7900e81dee24a5cce is 50, key is test_row_0/C:col10/1731577433360/Put/seqid=0 2024-11-14T09:43:54,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741935_1111 (size=12301) 2024-11-14T09:43:54,986 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/3b10d1882a2d49b7900e81dee24a5cce 2024-11-14T09:43:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/a388ad97475f4bf5bb334a0c7b40d665 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a388ad97475f4bf5bb334a0c7b40d665 2024-11-14T09:43:55,002 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a388ad97475f4bf5bb334a0c7b40d665, entries=150, sequenceid=618, filesize=12.0 K 2024-11-14T09:43:55,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/10706bcb5432403b9afe3c1c1a977112 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/10706bcb5432403b9afe3c1c1a977112 2024-11-14T09:43:55,009 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/10706bcb5432403b9afe3c1c1a977112, entries=150, sequenceid=618, filesize=12.0 K 2024-11-14T09:43:55,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/3b10d1882a2d49b7900e81dee24a5cce as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/3b10d1882a2d49b7900e81dee24a5cce 2024-11-14T09:43:55,019 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/3b10d1882a2d49b7900e81dee24a5cce, entries=150, sequenceid=618, filesize=12.0 K 2024-11-14T09:43:55,020 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~118.53 KB/121370, heapSize ~335.33 KB/343376, currentSize=60.38 KB/61830 for 52ef1722f0e126ebd82cd709b2dcdf9b in 650ms, sequenceid=618, compaction requested=false 2024-11-14T09:43:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:55,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T09:43:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T09:43:55,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T09:43:55,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 807 msec 2024-11-14T09:43:55,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 824 msec 2024-11-14T09:43:55,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:43:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:55,213 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T09:43:55,213 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,213 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T09:43:55,213 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/bb1a0e9106974f7a800c0d051824170f is 50, key is test_row_0/A:col10/1731577435184/Put/seqid=0 2024-11-14T09:43:55,215 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T09:43:55,215 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,217 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T09:43:55,217 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,219 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T09:43:55,219 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,221 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,221 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,221 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,221 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,221 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,221 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,221 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,221 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,222 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,222 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,227 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T09:43:55,227 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,227 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,227 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,229 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T09:43:55,229 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,229 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,229 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741936_1112 (size=12301) 2024-11-14T09:43:55,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/bb1a0e9106974f7a800c0d051824170f 2024-11-14T09:43:55,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/48453a511d7c4bdbaac5428a778e7aa1 is 50, key is test_row_0/B:col10/1731577435184/Put/seqid=0 2024-11-14T09:43:55,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741937_1113 (size=12301) 2024-11-14T09:43:55,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/48453a511d7c4bdbaac5428a778e7aa1 2024-11-14T09:43:55,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/260824d5a31c493ea0a0f919ee06f22b is 50, key is test_row_0/C:col10/1731577435184/Put/seqid=0 2024-11-14T09:43:55,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741938_1114 (size=12301) 2024-11-14T09:43:55,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/260824d5a31c493ea0a0f919ee06f22b 2024-11-14T09:43:55,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/bb1a0e9106974f7a800c0d051824170f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/bb1a0e9106974f7a800c0d051824170f 2024-11-14T09:43:55,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/bb1a0e9106974f7a800c0d051824170f, entries=150, sequenceid=634, filesize=12.0 K 2024-11-14T09:43:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/48453a511d7c4bdbaac5428a778e7aa1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48453a511d7c4bdbaac5428a778e7aa1 2024-11-14T09:43:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:43:55,351 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:55,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48453a511d7c4bdbaac5428a778e7aa1, entries=150, sequenceid=634, filesize=12.0 K 2024-11-14T09:43:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/260824d5a31c493ea0a0f919ee06f22b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/260824d5a31c493ea0a0f919ee06f22b 2024-11-14T09:43:55,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/260824d5a31c493ea0a0f919ee06f22b, entries=150, sequenceid=634, filesize=12.0 K 2024-11-14T09:43:55,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=46.96 KB/48090 for 52ef1722f0e126ebd82cd709b2dcdf9b in 224ms, sequenceid=634, compaction requested=true 2024-11-14T09:43:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:43:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:43:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:55,417 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52ef1722f0e126ebd82cd709b2dcdf9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:43:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:55,417 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:55,418 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:55,418 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:55,418 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/A is initiating minor compaction (all files) 2024-11-14T09:43:55,418 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/B is initiating minor compaction (all files) 2024-11-14T09:43:55,419 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/B in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:55,419 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/03e2ff6cf03743309a5179156727bb66, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/10706bcb5432403b9afe3c1c1a977112, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48453a511d7c4bdbaac5428a778e7aa1] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=37.1 K 2024-11-14T09:43:55,419 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/A in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:55,419 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/26d1654147b9489a9fbf260d3ec38bcb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a388ad97475f4bf5bb334a0c7b40d665, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/bb1a0e9106974f7a800c0d051824170f] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=37.1 K 2024-11-14T09:43:55,419 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e2ff6cf03743309a5179156727bb66, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731577433207 2024-11-14T09:43:55,419 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26d1654147b9489a9fbf260d3ec38bcb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731577433207 2024-11-14T09:43:55,420 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 10706bcb5432403b9afe3c1c1a977112, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731577433338 2024-11-14T09:43:55,420 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting a388ad97475f4bf5bb334a0c7b40d665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731577433338 2024-11-14T09:43:55,420 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1a0e9106974f7a800c0d051824170f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731577434527 2024-11-14T09:43:55,420 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 48453a511d7c4bdbaac5428a778e7aa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731577434527 2024-11-14T09:43:55,445 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#A#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:55,446 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b8e164d06fa74e278c745499875338f5 is 50, key is test_row_0/A:col10/1731577435184/Put/seqid=0 2024-11-14T09:43:55,458 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:55,459 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/b5c8a193c6ab42479cbb0148267fd930 is 50, key is test_row_0/B:col10/1731577435184/Put/seqid=0 2024-11-14T09:43:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:55,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:43:55,463 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T09:43:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:55,463 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,463 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:55,463 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:55,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741939_1115 (size=13459) 2024-11-14T09:43:55,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/ed254d8477f54d9fbb93ac162b7ed56f is 50, key is test_row_0/A:col10/1731577435203/Put/seqid=0 2024-11-14T09:43:55,494 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/b8e164d06fa74e278c745499875338f5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b8e164d06fa74e278c745499875338f5 2024-11-14T09:43:55,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741940_1116 (size=13459) 2024-11-14T09:43:55,507 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/A of 52ef1722f0e126ebd82cd709b2dcdf9b into b8e164d06fa74e278c745499875338f5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:55,507 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:55,507 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/A, priority=13, startTime=1731577435417; duration=0sec 2024-11-14T09:43:55,507 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:43:55,508 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:A 2024-11-14T09:43:55,508 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:43:55,513 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:43:55,513 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 52ef1722f0e126ebd82cd709b2dcdf9b/C is initiating minor compaction (all files) 2024-11-14T09:43:55,514 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52ef1722f0e126ebd82cd709b2dcdf9b/C in TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:55,514 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9503c1904f804a37a3a94b38a0f2f010, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/3b10d1882a2d49b7900e81dee24a5cce, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/260824d5a31c493ea0a0f919ee06f22b] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp, totalSize=37.1 K 2024-11-14T09:43:55,515 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9503c1904f804a37a3a94b38a0f2f010, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731577433207 2024-11-14T09:43:55,515 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b10d1882a2d49b7900e81dee24a5cce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1731577433338 2024-11-14T09:43:55,516 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260824d5a31c493ea0a0f919ee06f22b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731577434527 2024-11-14T09:43:55,517 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/b5c8a193c6ab42479cbb0148267fd930 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/b5c8a193c6ab42479cbb0148267fd930 2024-11-14T09:43:55,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741941_1117 (size=12301) 2024-11-14T09:43:55,538 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/B of 52ef1722f0e126ebd82cd709b2dcdf9b into b5c8a193c6ab42479cbb0148267fd930(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:55,538 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:55,538 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/B, priority=13, startTime=1731577435417; duration=0sec 2024-11-14T09:43:55,538 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:55,538 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:B 2024-11-14T09:43:55,545 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52ef1722f0e126ebd82cd709b2dcdf9b#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:43:55,546 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c50e420c40464065b91cce0ba096ea31 is 50, key is test_row_0/C:col10/1731577435184/Put/seqid=0 2024-11-14T09:43:55,572 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T09:43:55,572 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:43:55,572 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,572 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2145 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2238 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1035 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3096 rows 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1035 2024-11-14T09:43:55,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3098 rows 2024-11-14T09:43:55,573 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:43:55,574 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:43:55,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:43:55,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:43:55,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:43:55,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741942_1118 (size=13459) 2024-11-14T09:43:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:55,593 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/c50e420c40464065b91cce0ba096ea31 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c50e420c40464065b91cce0ba096ea31 2024-11-14T09:43:55,593 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577435592"}]},"ts":"1731577435592"} 2024-11-14T09:43:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:43:55,597 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:43:55,597 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:43:55,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:43:55,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, UNASSIGN}] 2024-11-14T09:43:55,610 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52ef1722f0e126ebd82cd709b2dcdf9b/C of 52ef1722f0e126ebd82cd709b2dcdf9b into c50e420c40464065b91cce0ba096ea31(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:43:55,610 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:55,610 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., storeName=52ef1722f0e126ebd82cd709b2dcdf9b/C, priority=13, startTime=1731577435417; duration=0sec 2024-11-14T09:43:55,610 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:43:55,610 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, UNASSIGN 2024-11-14T09:43:55,610 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52ef1722f0e126ebd82cd709b2dcdf9b:C 2024-11-14T09:43:55,613 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=52ef1722f0e126ebd82cd709b2dcdf9b, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:43:55,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, UNASSIGN because future has completed 2024-11-14T09:43:55,619 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:43:55,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:43:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:43:55,781 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:55,781 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:43:55,782 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 52ef1722f0e126ebd82cd709b2dcdf9b, disabling compactions & flushes 2024-11-14T09:43:55,782 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:43:55,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/ed254d8477f54d9fbb93ac162b7ed56f 2024-11-14T09:43:55,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/868d6272d33943c296bb68454e05169a is 50, key is test_row_0/B:col10/1731577435203/Put/seqid=0 2024-11-14T09:43:55,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741943_1119 (size=12301) 2024-11-14T09:43:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:43:56,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/868d6272d33943c296bb68454e05169a 2024-11-14T09:43:56,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/7a15929895d44573857939131e18ff90 is 50, key is test_row_0/C:col10/1731577435203/Put/seqid=0 2024-11-14T09:43:56,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741944_1120 (size=12301) 2024-11-14T09:43:56,642 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:43:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:43:56,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/7a15929895d44573857939131e18ff90 2024-11-14T09:43:56,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/ed254d8477f54d9fbb93ac162b7ed56f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/ed254d8477f54d9fbb93ac162b7ed56f 2024-11-14T09:43:56,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/ed254d8477f54d9fbb93ac162b7ed56f, entries=150, sequenceid=645, filesize=12.0 K 2024-11-14T09:43:56,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/868d6272d33943c296bb68454e05169a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/868d6272d33943c296bb68454e05169a 2024-11-14T09:43:56,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/868d6272d33943c296bb68454e05169a, entries=150, sequenceid=645, filesize=12.0 K 2024-11-14T09:43:56,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/7a15929895d44573857939131e18ff90 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7a15929895d44573857939131e18ff90 2024-11-14T09:43:56,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7a15929895d44573857939131e18ff90, entries=150, sequenceid=645, filesize=12.0 K 2024-11-14T09:43:56,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=6.71 KB/6870 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1336ms, sequenceid=645, compaction requested=false 2024-11-14T09:43:56,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52ef1722f0e126ebd82cd709b2dcdf9b: 2024-11-14T09:43:56,799 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:56,799 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:56,799 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. after waiting 0 ms 2024-11-14T09:43:56,799 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:56,799 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 52ef1722f0e126ebd82cd709b2dcdf9b 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-14T09:43:56,799 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=A 2024-11-14T09:43:56,800 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:56,800 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=B 2024-11-14T09:43:56,800 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:56,800 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52ef1722f0e126ebd82cd709b2dcdf9b, store=C 2024-11-14T09:43:56,800 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:43:56,804 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/9d49fb2c47c44b409b0b6f91dea1dab7 is 50, key is test_row_2/A:col10/1731577435570/Put/seqid=0 2024-11-14T09:43:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741945_1121 (size=7415) 2024-11-14T09:43:57,211 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=652 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/9d49fb2c47c44b409b0b6f91dea1dab7 2024-11-14T09:43:57,219 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/26f68638b3664f03af098d5142fb52cd is 50, key is test_row_2/B:col10/1731577435570/Put/seqid=0 2024-11-14T09:43:57,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741946_1122 (size=7415) 2024-11-14T09:43:57,625 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=652 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/26f68638b3664f03af098d5142fb52cd 2024-11-14T09:43:57,636 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b559b730aca4432980b01cc11e66b7e2 is 50, key is test_row_2/C:col10/1731577435570/Put/seqid=0 2024-11-14T09:43:57,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741947_1123 (size=7415) 2024-11-14T09:43:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:43:58,041 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=652 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b559b730aca4432980b01cc11e66b7e2 2024-11-14T09:43:58,047 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/A/9d49fb2c47c44b409b0b6f91dea1dab7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/9d49fb2c47c44b409b0b6f91dea1dab7 2024-11-14T09:43:58,054 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/9d49fb2c47c44b409b0b6f91dea1dab7, entries=50, sequenceid=652, filesize=7.2 K 2024-11-14T09:43:58,055 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/B/26f68638b3664f03af098d5142fb52cd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/26f68638b3664f03af098d5142fb52cd 2024-11-14T09:43:58,061 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/26f68638b3664f03af098d5142fb52cd, entries=50, sequenceid=652, filesize=7.2 K 2024-11-14T09:43:58,062 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/.tmp/C/b559b730aca4432980b01cc11e66b7e2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b559b730aca4432980b01cc11e66b7e2 2024-11-14T09:43:58,067 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b559b730aca4432980b01cc11e66b7e2, entries=50, sequenceid=652, filesize=7.2 K 2024-11-14T09:43:58,068 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1269ms, sequenceid=652, compaction requested=true 2024-11-14T09:43:58,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/93910310fcec475e8a50ef5b848d2c32, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b50c918a1c0648f79b4054923644a0ec, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/d3e15164e91f4d7ea45697657964920f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/168d01899246428d8c7c6f690260f928, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e143070bc81f4db79c4a15a13dd06a17, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c2b67d375c404500b55a31de35f890e1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e67b10970add4ccfb85da9488d18ddcb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c4d87311f92e43aa8a2450666f561453, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/088c5a2d1708445d968cd698578b1f72, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/939af23c53c24e3f9485f403d74855d7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/940c5e78ca6b4201b1a4181713053d17, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/541b677fe22b43278d4646a71a367fe0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/95d164ef01134f76964cb2265cd90e53, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/97f8c77027504c75b36b4f49eb9656eb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/48ea1eeb97a640e9ac52200b8e101115, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/3ea98686d71b4a39a5223d15caccd500, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b4883e6f1b7e46ccb5a579e809f18507, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/383df07bd7734556aa82a4871214bc22, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8426071272b247229e7084e00b982fd9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0e61fae381b846398a3bffbef1b00198, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/23ba956de66b416986de865bc337f7bd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0b0ea6b84ab54f65b033ce4931f8bb83, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/2976cf81840049198ca05ff4ae8a4b7d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/30a22ea6d55040e1acfe5e836f364d4e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a8648044903740f28e1206614d4cff76, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/36dbc0a38a544398b673017472457bac, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c872ef30159e4bb2a37af63d4d870dfd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c3700a7ca4da425b9309f5e5d8cb0edc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/41542b17d44540f781606b88c5db3980, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8a38b87643d14f17ae8b40ae4b26ff94, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/26d1654147b9489a9fbf260d3ec38bcb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8c1f5fd475384479a4f912809c89c181, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a388ad97475f4bf5bb334a0c7b40d665, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/bb1a0e9106974f7a800c0d051824170f] to archive 2024-11-14T09:43:58,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:43:58,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/93910310fcec475e8a50ef5b848d2c32 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/93910310fcec475e8a50ef5b848d2c32 2024-11-14T09:43:58,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b50c918a1c0648f79b4054923644a0ec to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b50c918a1c0648f79b4054923644a0ec 2024-11-14T09:43:58,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/d3e15164e91f4d7ea45697657964920f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/d3e15164e91f4d7ea45697657964920f 2024-11-14T09:43:58,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/168d01899246428d8c7c6f690260f928 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/168d01899246428d8c7c6f690260f928 2024-11-14T09:43:58,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e143070bc81f4db79c4a15a13dd06a17 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e143070bc81f4db79c4a15a13dd06a17 2024-11-14T09:43:58,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c2b67d375c404500b55a31de35f890e1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c2b67d375c404500b55a31de35f890e1 2024-11-14T09:43:58,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e67b10970add4ccfb85da9488d18ddcb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/e67b10970add4ccfb85da9488d18ddcb 2024-11-14T09:43:58,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c4d87311f92e43aa8a2450666f561453 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c4d87311f92e43aa8a2450666f561453 2024-11-14T09:43:58,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/088c5a2d1708445d968cd698578b1f72 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/088c5a2d1708445d968cd698578b1f72 2024-11-14T09:43:58,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/939af23c53c24e3f9485f403d74855d7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/939af23c53c24e3f9485f403d74855d7 2024-11-14T09:43:58,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/940c5e78ca6b4201b1a4181713053d17 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/940c5e78ca6b4201b1a4181713053d17 2024-11-14T09:43:58,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/541b677fe22b43278d4646a71a367fe0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/541b677fe22b43278d4646a71a367fe0 2024-11-14T09:43:58,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/95d164ef01134f76964cb2265cd90e53 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/95d164ef01134f76964cb2265cd90e53 2024-11-14T09:43:58,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/97f8c77027504c75b36b4f49eb9656eb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/97f8c77027504c75b36b4f49eb9656eb 2024-11-14T09:43:58,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/48ea1eeb97a640e9ac52200b8e101115 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/48ea1eeb97a640e9ac52200b8e101115 2024-11-14T09:43:58,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/3ea98686d71b4a39a5223d15caccd500 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/3ea98686d71b4a39a5223d15caccd500 2024-11-14T09:43:58,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b4883e6f1b7e46ccb5a579e809f18507 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b4883e6f1b7e46ccb5a579e809f18507 2024-11-14T09:43:58,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/383df07bd7734556aa82a4871214bc22 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/383df07bd7734556aa82a4871214bc22 2024-11-14T09:43:58,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8426071272b247229e7084e00b982fd9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8426071272b247229e7084e00b982fd9 2024-11-14T09:43:58,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0e61fae381b846398a3bffbef1b00198 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0e61fae381b846398a3bffbef1b00198 2024-11-14T09:43:58,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/23ba956de66b416986de865bc337f7bd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/23ba956de66b416986de865bc337f7bd 2024-11-14T09:43:58,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0b0ea6b84ab54f65b033ce4931f8bb83 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/0b0ea6b84ab54f65b033ce4931f8bb83 2024-11-14T09:43:58,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/2976cf81840049198ca05ff4ae8a4b7d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/2976cf81840049198ca05ff4ae8a4b7d 2024-11-14T09:43:58,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/30a22ea6d55040e1acfe5e836f364d4e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/30a22ea6d55040e1acfe5e836f364d4e 2024-11-14T09:43:58,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a8648044903740f28e1206614d4cff76 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a8648044903740f28e1206614d4cff76 2024-11-14T09:43:58,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/36dbc0a38a544398b673017472457bac to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/36dbc0a38a544398b673017472457bac 2024-11-14T09:43:58,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c872ef30159e4bb2a37af63d4d870dfd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c872ef30159e4bb2a37af63d4d870dfd 2024-11-14T09:43:58,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c3700a7ca4da425b9309f5e5d8cb0edc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/c3700a7ca4da425b9309f5e5d8cb0edc 2024-11-14T09:43:58,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/41542b17d44540f781606b88c5db3980 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/41542b17d44540f781606b88c5db3980 2024-11-14T09:43:58,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8a38b87643d14f17ae8b40ae4b26ff94 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8a38b87643d14f17ae8b40ae4b26ff94 2024-11-14T09:43:58,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/26d1654147b9489a9fbf260d3ec38bcb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/26d1654147b9489a9fbf260d3ec38bcb 2024-11-14T09:43:58,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8c1f5fd475384479a4f912809c89c181 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/8c1f5fd475384479a4f912809c89c181 2024-11-14T09:43:58,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a388ad97475f4bf5bb334a0c7b40d665 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/a388ad97475f4bf5bb334a0c7b40d665 2024-11-14T09:43:58,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/bb1a0e9106974f7a800c0d051824170f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/bb1a0e9106974f7a800c0d051824170f 2024-11-14T09:43:58,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e7b067cf71a54f27bb13fdf5192c665a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/7e11cdc279ad47e8a3af3096b82550a8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/51a5b2c7c836456b9e6a01f1e4c2e950, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ce7407afbe37403bbb26dc45f303efb6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e793f711ee9a458cb1c1b895fb2e123d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ef304ea3795a4e4791cadc67e62a7509, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48b9f73a937d46e2831198e27e80430a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9ba109845864ce792cd3fadef560be7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4eaefec4a1a44ee28d748e71c0d5dd9e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/a2a155e3e5eb4e68ae9abf10d8a92b71, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d8c357222ad5451d88892d44994a41f7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4fd7805e984949bdab5a34e69e52aadb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9459d068d2f40f5a3f15cfe7c52903b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9ae176060a654fadbb73e80358608508, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/46e2ed46f473414dac0e4379370d5c0d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/3f1843a2bc4a47729728d9e776626293, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d4f62a31e7994f55bf5f785bd87e6acf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4607c180216b413e93ca997255be1417, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ca0074bb1bdf47f8a963ccdb49b7f007, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/12183cea449b42c0ab97b1bc0efd5a5c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e87bc620191c4a879d0b469897e210f6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4a6f89fcc37741148390968d96e621fe, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/cec276104af340779a40c7ee9d3b18f8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/fd1ad96ba4054e2ab189b69efb76433d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/21d0f5cc22484e68baf574b72916937b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9e93c3bf88744d00928f06099223b31c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c4a65fbb0de14fc88632091d2ba2e781, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6a9960b50bed400787bb5d9f9b0f0526, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c43dc75170964959b0033e403954ec5b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6dea3ccaec1f48d4adb82e80567b1ed0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/03e2ff6cf03743309a5179156727bb66, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/08ec8b687e014f27b102b10d9885eb5e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/10706bcb5432403b9afe3c1c1a977112, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48453a511d7c4bdbaac5428a778e7aa1] to archive 2024-11-14T09:43:58,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:43:58,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e7b067cf71a54f27bb13fdf5192c665a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e7b067cf71a54f27bb13fdf5192c665a 2024-11-14T09:43:58,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/7e11cdc279ad47e8a3af3096b82550a8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/7e11cdc279ad47e8a3af3096b82550a8 2024-11-14T09:43:58,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/51a5b2c7c836456b9e6a01f1e4c2e950 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/51a5b2c7c836456b9e6a01f1e4c2e950 2024-11-14T09:43:58,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ce7407afbe37403bbb26dc45f303efb6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ce7407afbe37403bbb26dc45f303efb6 2024-11-14T09:43:58,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e793f711ee9a458cb1c1b895fb2e123d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e793f711ee9a458cb1c1b895fb2e123d 2024-11-14T09:43:58,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ef304ea3795a4e4791cadc67e62a7509 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ef304ea3795a4e4791cadc67e62a7509 2024-11-14T09:43:58,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48b9f73a937d46e2831198e27e80430a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48b9f73a937d46e2831198e27e80430a 2024-11-14T09:43:58,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9ba109845864ce792cd3fadef560be7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9ba109845864ce792cd3fadef560be7 2024-11-14T09:43:58,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4eaefec4a1a44ee28d748e71c0d5dd9e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4eaefec4a1a44ee28d748e71c0d5dd9e 2024-11-14T09:43:58,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/a2a155e3e5eb4e68ae9abf10d8a92b71 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/a2a155e3e5eb4e68ae9abf10d8a92b71 2024-11-14T09:43:58,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d8c357222ad5451d88892d44994a41f7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d8c357222ad5451d88892d44994a41f7 2024-11-14T09:43:58,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4fd7805e984949bdab5a34e69e52aadb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4fd7805e984949bdab5a34e69e52aadb 2024-11-14T09:43:58,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9459d068d2f40f5a3f15cfe7c52903b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/f9459d068d2f40f5a3f15cfe7c52903b 2024-11-14T09:43:58,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9ae176060a654fadbb73e80358608508 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9ae176060a654fadbb73e80358608508 2024-11-14T09:43:58,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/46e2ed46f473414dac0e4379370d5c0d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/46e2ed46f473414dac0e4379370d5c0d 2024-11-14T09:43:58,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/3f1843a2bc4a47729728d9e776626293 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/3f1843a2bc4a47729728d9e776626293 2024-11-14T09:43:58,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d4f62a31e7994f55bf5f785bd87e6acf to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/d4f62a31e7994f55bf5f785bd87e6acf 2024-11-14T09:43:58,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4607c180216b413e93ca997255be1417 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4607c180216b413e93ca997255be1417 2024-11-14T09:43:58,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ca0074bb1bdf47f8a963ccdb49b7f007 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/ca0074bb1bdf47f8a963ccdb49b7f007 2024-11-14T09:43:58,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/12183cea449b42c0ab97b1bc0efd5a5c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/12183cea449b42c0ab97b1bc0efd5a5c 2024-11-14T09:43:58,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e87bc620191c4a879d0b469897e210f6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/e87bc620191c4a879d0b469897e210f6 2024-11-14T09:43:58,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4a6f89fcc37741148390968d96e621fe to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/4a6f89fcc37741148390968d96e621fe 2024-11-14T09:43:58,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/cec276104af340779a40c7ee9d3b18f8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/cec276104af340779a40c7ee9d3b18f8 2024-11-14T09:43:58,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/fd1ad96ba4054e2ab189b69efb76433d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/fd1ad96ba4054e2ab189b69efb76433d 2024-11-14T09:43:58,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/21d0f5cc22484e68baf574b72916937b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/21d0f5cc22484e68baf574b72916937b 2024-11-14T09:43:58,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9e93c3bf88744d00928f06099223b31c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/9e93c3bf88744d00928f06099223b31c 2024-11-14T09:43:58,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c4a65fbb0de14fc88632091d2ba2e781 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c4a65fbb0de14fc88632091d2ba2e781 2024-11-14T09:43:58,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6a9960b50bed400787bb5d9f9b0f0526 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6a9960b50bed400787bb5d9f9b0f0526 2024-11-14T09:43:58,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c43dc75170964959b0033e403954ec5b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/c43dc75170964959b0033e403954ec5b 2024-11-14T09:43:58,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6dea3ccaec1f48d4adb82e80567b1ed0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/6dea3ccaec1f48d4adb82e80567b1ed0 2024-11-14T09:43:58,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/03e2ff6cf03743309a5179156727bb66 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/03e2ff6cf03743309a5179156727bb66 2024-11-14T09:43:58,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/08ec8b687e014f27b102b10d9885eb5e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/08ec8b687e014f27b102b10d9885eb5e 2024-11-14T09:43:58,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/10706bcb5432403b9afe3c1c1a977112 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/10706bcb5432403b9afe3c1c1a977112 2024-11-14T09:43:58,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48453a511d7c4bdbaac5428a778e7aa1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/48453a511d7c4bdbaac5428a778e7aa1 2024-11-14T09:43:58,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b24ff9d6cb084377bc82cd5812ebab96, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ec03f68ca0f74186a621b4f230d10938, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7d0bedf2ea7e43f69de157528206dc7a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/0ee0a25198044d378cec1b91f39821ac, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8105d63eca334d1ca5d5026cd4df76db, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b8bf714cb7234ebaa4a7b151be6a68dd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/bd27f9875dc047cf9bcf3551f38e326d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5516335bb87e49fba8c4fa60777b854d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9ebee544f03645eb84f4561040c4bccc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/82114857f6694099be97e8208d0b08bb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/35f85c7ea70b4eb9a9c917b87d6d7921, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/2059c498de5642cdbf9a3495e2283b75, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/1fdf9957698f42ca887fa471c7a2aecf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/40e88b9357f44873a5d1854d20327479, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c621da6b3cf14564ae62ce06f788b1e4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5aae49feb2864624909e8dafdcaabb67, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/d25d3ecf6b0f42a7a9717260efb8e358, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/789a1efe7d244674a6b6d5fd0eb508e6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/534ddf1d3d63464993161c0a2a4dd763, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/259c4ece0f76408990b4d5a75959a9d0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b55f4742d4084194a6bd75140b17acdf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/14a4a9511d76469fb63086ba550c21c1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/33ade06324534ba3902f12be2765968f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ad9357f252634b13b976177e3f2a5934, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8ef3aadebe0c4995b579512f987a5369, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/090ccd7e7e65487aa4056cc4dcc22252, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c91310cf2ae6493091a857352affb6ed, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/48d4f005db5b4f2aabf6c8d28578f3d9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/4daadf7edeb04b62816ba8cfe79c07c5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/07bd75d7ea5c4459820b2c522001a54e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9503c1904f804a37a3a94b38a0f2f010, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8af8daabebc6427993d96a7578d5c500, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/3b10d1882a2d49b7900e81dee24a5cce, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/260824d5a31c493ea0a0f919ee06f22b] to archive 2024-11-14T09:43:58,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:43:58,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b24ff9d6cb084377bc82cd5812ebab96 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b24ff9d6cb084377bc82cd5812ebab96 2024-11-14T09:43:58,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ec03f68ca0f74186a621b4f230d10938 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ec03f68ca0f74186a621b4f230d10938 2024-11-14T09:43:58,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7d0bedf2ea7e43f69de157528206dc7a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7d0bedf2ea7e43f69de157528206dc7a 2024-11-14T09:43:58,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/0ee0a25198044d378cec1b91f39821ac to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/0ee0a25198044d378cec1b91f39821ac 2024-11-14T09:43:58,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8105d63eca334d1ca5d5026cd4df76db to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8105d63eca334d1ca5d5026cd4df76db 2024-11-14T09:43:58,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b8bf714cb7234ebaa4a7b151be6a68dd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b8bf714cb7234ebaa4a7b151be6a68dd 2024-11-14T09:43:58,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/bd27f9875dc047cf9bcf3551f38e326d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/bd27f9875dc047cf9bcf3551f38e326d 2024-11-14T09:43:58,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5516335bb87e49fba8c4fa60777b854d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5516335bb87e49fba8c4fa60777b854d 2024-11-14T09:43:58,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9ebee544f03645eb84f4561040c4bccc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9ebee544f03645eb84f4561040c4bccc 2024-11-14T09:43:58,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/82114857f6694099be97e8208d0b08bb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/82114857f6694099be97e8208d0b08bb 2024-11-14T09:43:58,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/35f85c7ea70b4eb9a9c917b87d6d7921 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/35f85c7ea70b4eb9a9c917b87d6d7921 2024-11-14T09:43:58,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/2059c498de5642cdbf9a3495e2283b75 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/2059c498de5642cdbf9a3495e2283b75 2024-11-14T09:43:58,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/1fdf9957698f42ca887fa471c7a2aecf to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/1fdf9957698f42ca887fa471c7a2aecf 2024-11-14T09:43:58,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/40e88b9357f44873a5d1854d20327479 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/40e88b9357f44873a5d1854d20327479 2024-11-14T09:43:58,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c621da6b3cf14564ae62ce06f788b1e4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c621da6b3cf14564ae62ce06f788b1e4 2024-11-14T09:43:58,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5aae49feb2864624909e8dafdcaabb67 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/5aae49feb2864624909e8dafdcaabb67 2024-11-14T09:43:58,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/d25d3ecf6b0f42a7a9717260efb8e358 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/d25d3ecf6b0f42a7a9717260efb8e358 2024-11-14T09:43:58,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/789a1efe7d244674a6b6d5fd0eb508e6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/789a1efe7d244674a6b6d5fd0eb508e6 2024-11-14T09:43:58,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/534ddf1d3d63464993161c0a2a4dd763 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/534ddf1d3d63464993161c0a2a4dd763 2024-11-14T09:43:58,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/259c4ece0f76408990b4d5a75959a9d0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/259c4ece0f76408990b4d5a75959a9d0 2024-11-14T09:43:58,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b55f4742d4084194a6bd75140b17acdf to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b55f4742d4084194a6bd75140b17acdf 2024-11-14T09:43:58,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/14a4a9511d76469fb63086ba550c21c1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/14a4a9511d76469fb63086ba550c21c1 2024-11-14T09:43:58,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/33ade06324534ba3902f12be2765968f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/33ade06324534ba3902f12be2765968f 2024-11-14T09:43:58,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ad9357f252634b13b976177e3f2a5934 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/ad9357f252634b13b976177e3f2a5934 2024-11-14T09:43:58,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8ef3aadebe0c4995b579512f987a5369 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8ef3aadebe0c4995b579512f987a5369 2024-11-14T09:43:58,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/090ccd7e7e65487aa4056cc4dcc22252 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/090ccd7e7e65487aa4056cc4dcc22252 2024-11-14T09:43:58,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c91310cf2ae6493091a857352affb6ed to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c91310cf2ae6493091a857352affb6ed 2024-11-14T09:43:58,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/48d4f005db5b4f2aabf6c8d28578f3d9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/48d4f005db5b4f2aabf6c8d28578f3d9 2024-11-14T09:43:58,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/4daadf7edeb04b62816ba8cfe79c07c5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/4daadf7edeb04b62816ba8cfe79c07c5 2024-11-14T09:43:58,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/07bd75d7ea5c4459820b2c522001a54e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/07bd75d7ea5c4459820b2c522001a54e 2024-11-14T09:43:58,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9503c1904f804a37a3a94b38a0f2f010 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/9503c1904f804a37a3a94b38a0f2f010 2024-11-14T09:43:58,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8af8daabebc6427993d96a7578d5c500 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/8af8daabebc6427993d96a7578d5c500 2024-11-14T09:43:58,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/3b10d1882a2d49b7900e81dee24a5cce to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/3b10d1882a2d49b7900e81dee24a5cce 2024-11-14T09:43:58,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/260824d5a31c493ea0a0f919ee06f22b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/260824d5a31c493ea0a0f919ee06f22b 2024-11-14T09:43:58,213 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/recovered.edits/655.seqid, newMaxSeqId=655, maxSeqId=1 2024-11-14T09:43:58,214 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. 2024-11-14T09:43:58,214 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 52ef1722f0e126ebd82cd709b2dcdf9b: Waiting for close lock at 1731577435782Running coprocessor pre-close hooks at 1731577435782Disabling compacts and flushes for region at 1731577435782Disabling writes for close at 1731577436799 (+1017 ms)Obtaining lock to block concurrent updates at 1731577436799Preparing flush snapshotting stores in 52ef1722f0e126ebd82cd709b2dcdf9b at 1731577436799Finished memstore snapshotting TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1731577436800 (+1 ms)Flushing stores of TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b. at 1731577436800Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/A: creating writer at 1731577436800Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/A: appending metadata at 1731577436803 (+3 ms)Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/A: closing flushed file at 1731577436803Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/B: creating writer at 1731577437216 (+413 ms)Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/B: appending metadata at 1731577437219 (+3 ms)Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/B: closing flushed file at 1731577437219Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/C: creating writer at 1731577437633 (+414 ms)Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/C: appending metadata at 1731577437636 (+3 ms)Flushing 52ef1722f0e126ebd82cd709b2dcdf9b/C: closing flushed file at 1731577437636Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@655a3194: reopening flushed file at 1731577438046 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c085212: reopening flushed file at 1731577438054 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c8beea3: reopening flushed file at 1731577438061 (+7 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 52ef1722f0e126ebd82cd709b2dcdf9b in 1269ms, sequenceid=652, compaction requested=true at 1731577438068 (+7 ms)Writing region close event to WAL at 1731577438209 (+141 ms)Running coprocessor post-close hooks at 1731577438214 (+5 ms)Closed at 1731577438214 2024-11-14T09:43:58,217 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:58,218 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=52ef1722f0e126ebd82cd709b2dcdf9b, regionState=CLOSED 2024-11-14T09:43:58,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:43:58,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T09:43:58,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 52ef1722f0e126ebd82cd709b2dcdf9b, server=defc576eb6b7,42511,1731577408298 in 2.6020 sec 2024-11-14T09:43:58,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-14T09:43:58,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52ef1722f0e126ebd82cd709b2dcdf9b, UNASSIGN in 2.6150 sec 2024-11-14T09:43:58,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T09:43:58,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6260 sec 2024-11-14T09:43:58,229 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577438229"}]},"ts":"1731577438229"} 2024-11-14T09:43:58,231 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:43:58,231 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:43:58,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6500 sec 2024-11-14T09:43:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:43:59,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:59,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:43:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:59,773 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:43:59,775 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:59,780 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:59,783 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/recovered.edits] 2024-11-14T09:43:59,786 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/9d49fb2c47c44b409b0b6f91dea1dab7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/9d49fb2c47c44b409b0b6f91dea1dab7 2024-11-14T09:43:59,788 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b8e164d06fa74e278c745499875338f5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/b8e164d06fa74e278c745499875338f5 2024-11-14T09:43:59,789 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/ed254d8477f54d9fbb93ac162b7ed56f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/A/ed254d8477f54d9fbb93ac162b7ed56f 2024-11-14T09:43:59,791 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/26f68638b3664f03af098d5142fb52cd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/26f68638b3664f03af098d5142fb52cd 2024-11-14T09:43:59,793 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/868d6272d33943c296bb68454e05169a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/868d6272d33943c296bb68454e05169a 2024-11-14T09:43:59,794 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/b5c8a193c6ab42479cbb0148267fd930 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/B/b5c8a193c6ab42479cbb0148267fd930 2024-11-14T09:43:59,797 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7a15929895d44573857939131e18ff90 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/7a15929895d44573857939131e18ff90 2024-11-14T09:43:59,798 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b559b730aca4432980b01cc11e66b7e2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/b559b730aca4432980b01cc11e66b7e2 2024-11-14T09:43:59,800 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c50e420c40464065b91cce0ba096ea31 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/C/c50e420c40464065b91cce0ba096ea31 2024-11-14T09:43:59,804 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/recovered.edits/655.seqid to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b/recovered.edits/655.seqid 2024-11-14T09:43:59,804 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/52ef1722f0e126ebd82cd709b2dcdf9b 2024-11-14T09:43:59,804 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:43:59,809 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:59,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T09:43:59,821 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:43:59,856 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:43:59,858 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:59,858 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:43:59,858 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577439858"}]},"ts":"9223372036854775807"} 2024-11-14T09:43:59,863 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:43:59,863 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 52ef1722f0e126ebd82cd709b2dcdf9b, NAME => 'TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:43:59,864 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:43:59,864 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577439864"}]},"ts":"9223372036854775807"} 2024-11-14T09:43:59,867 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:43:59,868 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:59,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-14T09:43:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:43:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:43:59,882 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:43:59,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;defc576eb6b7:42511-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-572166304_22 at /127.0.0.1:38056 [Waiting for operation #353] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-969773813_22 at /127.0.0.1:53448 [Waiting for operation #338] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=416 (was 165) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4082 (was 5148) 2024-11-14T09:43:59,903 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=416, ProcessCount=11, AvailableMemoryMB=4080 2024-11-14T09:43:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:43:59,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:43:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:43:59,906 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:43:59,906 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:43:59,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-14T09:43:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T09:43:59,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:43:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741948_1124 (size=1061) 2024-11-14T09:44:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T09:44:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T09:44:00,318 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:44:00,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741949_1125 (size=53) 2024-11-14T09:44:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T09:44:00,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:44:00,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 16957b2537beefdba492fe73e43233c2, disabling compactions & flushes 2024-11-14T09:44:00,730 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:00,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:00,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. after waiting 0 ms 2024-11-14T09:44:00,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:00,730 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:00,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 16957b2537beefdba492fe73e43233c2: Waiting for close lock at 1731577440730Disabling compacts and flushes for region at 1731577440730Disabling writes for close at 1731577440730Writing region close event to WAL at 1731577440730Closed at 1731577440730 2024-11-14T09:44:00,760 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:44:00,761 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577440760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577440760"}]},"ts":"1731577440760"} 2024-11-14T09:44:00,763 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:44:00,765 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:44:00,766 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577440765"}]},"ts":"1731577440765"} 2024-11-14T09:44:00,768 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:44:00,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, ASSIGN}] 2024-11-14T09:44:00,770 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, ASSIGN 2024-11-14T09:44:00,771 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, ASSIGN; state=OFFLINE, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=false 2024-11-14T09:44:00,922 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:00,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, ASSIGN because future has completed 2024-11-14T09:44:00,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:44:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T09:44:01,086 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:01,086 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:44:01,087 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,087 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:44:01,087 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,087 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,089 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,091 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:01,092 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16957b2537beefdba492fe73e43233c2 columnFamilyName A 2024-11-14T09:44:01,092 DEBUG [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:01,093 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(327): Store=16957b2537beefdba492fe73e43233c2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:01,093 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,094 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:01,094 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16957b2537beefdba492fe73e43233c2 columnFamilyName B 2024-11-14T09:44:01,094 DEBUG [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:01,095 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(327): Store=16957b2537beefdba492fe73e43233c2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:01,095 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,096 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:01,096 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16957b2537beefdba492fe73e43233c2 columnFamilyName C 2024-11-14T09:44:01,096 DEBUG [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:01,097 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(327): Store=16957b2537beefdba492fe73e43233c2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:01,097 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:01,097 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,098 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,098 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,099 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,099 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,100 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:44:01,101 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,103 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:44:01,103 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 16957b2537beefdba492fe73e43233c2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67108721, jitterRate=-2.130866050720215E-6}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:44:01,103 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:01,104 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 16957b2537beefdba492fe73e43233c2: Running coprocessor pre-open hook at 1731577441087Writing region info on filesystem at 1731577441087Initializing all the Stores at 1731577441088 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577441088Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577441088Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577441089 (+1 ms)Cleaning up temporary data from old regions at 1731577441099 (+10 ms)Running coprocessor post-open hooks at 1731577441103 (+4 ms)Region opened successfully at 1731577441104 (+1 ms) 2024-11-14T09:44:01,105 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., pid=28, masterSystemTime=1731577441076 2024-11-14T09:44:01,107 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:01,107 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:01,107 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:01,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:44:01,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T09:44:01,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 in 186 msec 2024-11-14T09:44:01,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-14T09:44:01,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, ASSIGN in 344 msec 2024-11-14T09:44:01,115 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:44:01,115 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577441115"}]},"ts":"1731577441115"} 2024-11-14T09:44:01,116 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:44:01,117 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:44:01,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2130 sec 2024-11-14T09:44:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T09:44:02,051 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:02,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:02,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:02,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:02,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2559f6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:02,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e15a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:02,057 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:02,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:02,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:44:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,33347,1731577407591 2024-11-14T09:44:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@66520796 2024-11-14T09:44:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:44:02,063 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:44:02,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:44:02,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:44:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:02,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741950_1126 (size=1097) 2024-11-14T09:44:02,493 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T09:44:02,494 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T09:44:02,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:44:02,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, REOPEN/MOVE}] 2024-11-14T09:44:02,507 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, REOPEN/MOVE 2024-11-14T09:44:02,508 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:02,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, REOPEN/MOVE because future has completed 2024-11-14T09:44:02,510 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:44:02,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:44:02,663 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:02,663 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:44:02,663 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 16957b2537beefdba492fe73e43233c2, disabling compactions & flushes 2024-11-14T09:44:02,663 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:02,663 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:02,663 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. after waiting 0 ms 2024-11-14T09:44:02,663 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:02,669 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:44:02,670 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:02,670 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 16957b2537beefdba492fe73e43233c2: Waiting for close lock at 1731577442663Running coprocessor pre-close hooks at 1731577442663Disabling compacts and flushes for region at 1731577442663Disabling writes for close at 1731577442663Writing region close event to WAL at 1731577442664 (+1 ms)Running coprocessor post-close hooks at 1731577442670 (+6 ms)Closed at 1731577442670 2024-11-14T09:44:02,670 WARN [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 16957b2537beefdba492fe73e43233c2 to self. 2024-11-14T09:44:02,673 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:02,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=CLOSED 2024-11-14T09:44:02,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:44:02,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T09:44:02,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 in 167 msec 2024-11-14T09:44:02,681 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, REOPEN/MOVE; state=CLOSED, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=true 2024-11-14T09:44:02,831 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:02,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, REOPEN/MOVE because future has completed 2024-11-14T09:44:02,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:44:02,993 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:02,993 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:44:02,994 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:02,994 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:44:02,994 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:02,995 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:02,999 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,000 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:03,004 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16957b2537beefdba492fe73e43233c2 columnFamilyName A 2024-11-14T09:44:03,006 DEBUG [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:03,006 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(327): Store=16957b2537beefdba492fe73e43233c2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:03,007 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,007 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:03,008 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16957b2537beefdba492fe73e43233c2 columnFamilyName B 2024-11-14T09:44:03,008 DEBUG [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:03,008 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(327): Store=16957b2537beefdba492fe73e43233c2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:03,008 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,009 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:03,009 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16957b2537beefdba492fe73e43233c2 columnFamilyName C 2024-11-14T09:44:03,009 DEBUG [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:03,010 INFO [StoreOpener-16957b2537beefdba492fe73e43233c2-1 {}] regionserver.HStore(327): Store=16957b2537beefdba492fe73e43233c2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:03,010 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,010 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,010 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,012 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,013 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,013 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,013 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:44:03,014 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,015 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 16957b2537beefdba492fe73e43233c2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66767192, jitterRate=-0.005091309547424316}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:44:03,015 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,016 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 16957b2537beefdba492fe73e43233c2: Running coprocessor pre-open hook at 1731577442995Writing region info on filesystem at 1731577442995Initializing all the Stores at 1731577442996 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577442996Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577442997 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577442997Cleaning up temporary data from old regions at 1731577443013 (+16 ms)Running coprocessor post-open hooks at 1731577443015 (+2 ms)Region opened successfully at 1731577443016 (+1 ms) 2024-11-14T09:44:03,017 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., pid=33, masterSystemTime=1731577442986 2024-11-14T09:44:03,019 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,019 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=OPEN, openSeqNum=5, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:44:03,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-14T09:44:03,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 in 189 msec 2024-11-14T09:44:03,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-14T09:44:03,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, REOPEN/MOVE in 519 msec 2024-11-14T09:44:03,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T09:44:03,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 529 msec 2024-11-14T09:44:03,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 960 msec 2024-11-14T09:44:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T09:44:03,036 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:03,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,038 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3898a5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,039 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b42a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,043 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e3e176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,046 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d33c78d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4e039f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,050 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52bd0d43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c41e5c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,053 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b5500f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,055 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@697eb372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6df16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15bf2458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,061 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed9c8f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76fa478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,065 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf70255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,067 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc3d081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:03,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:03,069 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4379173c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:03,070 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12988e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:03,071 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,071 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,071 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,072 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-14T09:44:03,073 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T09:44:03,074 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:03,074 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,074 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,075 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,076 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:03,076 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:03,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,077 DEBUG [Thread-641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,079 DEBUG [Thread-637 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:03,080 DEBUG [Thread-637 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:44:03,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:44:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:03,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111415cee01f25984be081a0309285e9c51a_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577443094/Put/seqid=0 2024-11-14T09:44:03,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741951_1127 (size=12154) 2024-11-14T09:44:03,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:03,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111415cee01f25984be081a0309285e9c51a_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111415cee01f25984be081a0309285e9c51a_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:03,160 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/58518b4d05334d61a2b76b2bb0bb740a, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:03,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577443121, max timestamp=1731577443151Num uniques 859; 2024-11-14T09:44:03,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:44:03,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577443121, max timestamp=1731577443151Num uniques 859; 2024-11-14T09:44:03,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:44:03,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/58518b4d05334d61a2b76b2bb0bb740a is 175, key is test_row_0/A:col10/1731577443094/Put/seqid=0 2024-11-14T09:44:03,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577443121, max timestamp=1731577443151Num uniques 859; 2024-11-14T09:44:03,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:44:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T09:44:03,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741952_1128 (size=30955) 2024-11-14T09:44:03,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577503190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577503192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577503203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577503207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577503209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T09:44:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577503320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577503320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577503320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577503330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577503330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T09:44:03,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:03,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T09:44:03,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577503530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T09:44:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577503541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577503541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577503541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577503541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,603 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/58518b4d05334d61a2b76b2bb0bb740a 2024-11-14T09:44:03,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/88a1b3abb0104edf9ecebf8bc5fd5e35 is 50, key is test_row_0/B:col10/1731577443094/Put/seqid=0 2024-11-14T09:44:03,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741953_1129 (size=12001) 2024-11-14T09:44:03,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/88a1b3abb0104edf9ecebf8bc5fd5e35 2024-11-14T09:44:03,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/58f399f4c97f4e6284e50403022187d1 is 50, key is test_row_0/C:col10/1731577443094/Put/seqid=0 2024-11-14T09:44:03,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741954_1130 (size=12001) 2024-11-14T09:44:03,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T09:44:03,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T09:44:03,774 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:44:03,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T09:44:03,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:03,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:03,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577503841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577503850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577503850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577503850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:03,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577503851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T09:44:04,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:04,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:04,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:04,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:04,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:04,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/58f399f4c97f4e6284e50403022187d1 2024-11-14T09:44:04,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/58518b4d05334d61a2b76b2bb0bb740a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/58518b4d05334d61a2b76b2bb0bb740a 2024-11-14T09:44:04,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/58518b4d05334d61a2b76b2bb0bb740a, entries=150, sequenceid=14, filesize=30.2 K 2024-11-14T09:44:04,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/88a1b3abb0104edf9ecebf8bc5fd5e35 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/88a1b3abb0104edf9ecebf8bc5fd5e35 2024-11-14T09:44:04,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/88a1b3abb0104edf9ecebf8bc5fd5e35, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:44:04,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/58f399f4c97f4e6284e50403022187d1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/58f399f4c97f4e6284e50403022187d1 2024-11-14T09:44:04,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/58f399f4c97f4e6284e50403022187d1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:44:04,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 16957b2537beefdba492fe73e43233c2 in 1034ms, sequenceid=14, compaction requested=false 2024-11-14T09:44:04,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:04,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T09:44:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:04,156 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-14T09:44:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:04,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b7133d421e234f49bb566e1b7e8815c4_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577443189/Put/seqid=0 2024-11-14T09:44:04,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741955_1131 (size=12154) 2024-11-14T09:44:04,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:04,192 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b7133d421e234f49bb566e1b7e8815c4_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b7133d421e234f49bb566e1b7e8815c4_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3dde710441074c8fbaf843969aaad4db, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3dde710441074c8fbaf843969aaad4db is 175, key is test_row_0/A:col10/1731577443189/Put/seqid=0 2024-11-14T09:44:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T09:44:04,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741956_1132 (size=30955) 2024-11-14T09:44:04,222 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3dde710441074c8fbaf843969aaad4db 2024-11-14T09:44:04,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/3169810c41dc49408bb22af78443c130 is 50, key is test_row_0/B:col10/1731577443189/Put/seqid=0 2024-11-14T09:44:04,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741957_1133 (size=12001) 2024-11-14T09:44:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:04,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:04,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577504368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577504368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577504369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577504370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577504372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577504480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577504480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577504480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577504481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577504481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,654 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/3169810c41dc49408bb22af78443c130 2024-11-14T09:44:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/02aa8bb257464a7183a8e74272171f43 is 50, key is test_row_0/C:col10/1731577443189/Put/seqid=0 2024-11-14T09:44:04,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741958_1134 (size=12001) 2024-11-14T09:44:04,679 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/02aa8bb257464a7183a8e74272171f43 2024-11-14T09:44:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3dde710441074c8fbaf843969aaad4db as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3dde710441074c8fbaf843969aaad4db 2024-11-14T09:44:04,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577504691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577504691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577504692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577504696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577504696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:04,702 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3dde710441074c8fbaf843969aaad4db, entries=150, sequenceid=48, filesize=30.2 K 2024-11-14T09:44:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/3169810c41dc49408bb22af78443c130 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/3169810c41dc49408bb22af78443c130 2024-11-14T09:44:04,713 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/3169810c41dc49408bb22af78443c130, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T09:44:04,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/02aa8bb257464a7183a8e74272171f43 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/02aa8bb257464a7183a8e74272171f43 2024-11-14T09:44:04,726 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/02aa8bb257464a7183a8e74272171f43, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T09:44:04,727 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 16957b2537beefdba492fe73e43233c2 in 571ms, sequenceid=48, compaction requested=false 2024-11-14T09:44:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:04,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-14T09:44:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-14T09:44:04,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-14T09:44:04,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6530 sec 2024-11-14T09:44:04,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.6600 sec 2024-11-14T09:44:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:05,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:44:05,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:05,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:05,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:05,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:05,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:05,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:05,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114da08131be94140bf9c15fad702636900_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577444371/Put/seqid=0 2024-11-14T09:44:05,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741959_1135 (size=12154) 2024-11-14T09:44:05,027 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:05,034 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114da08131be94140bf9c15fad702636900_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114da08131be94140bf9c15fad702636900_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:05,036 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/a069700ee8cb48699dd6d86166d86bf1, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:05,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/a069700ee8cb48699dd6d86166d86bf1 is 175, key is test_row_0/A:col10/1731577444371/Put/seqid=0 2024-11-14T09:44:05,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577505031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577505040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577505042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577505043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577505043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741960_1136 (size=30955) 2024-11-14T09:44:05,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/a069700ee8cb48699dd6d86166d86bf1 2024-11-14T09:44:05,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/658dca2c8a4b48f7b6418d3f5b58d83f is 50, key is test_row_0/B:col10/1731577444371/Put/seqid=0 2024-11-14T09:44:05,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741961_1137 (size=12001) 2024-11-14T09:44:05,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/658dca2c8a4b48f7b6418d3f5b58d83f 2024-11-14T09:44:05,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/112d9c25340b4686910b8b182fe5b263 is 50, key is test_row_0/C:col10/1731577444371/Put/seqid=0 2024-11-14T09:44:05,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741962_1138 (size=12001) 2024-11-14T09:44:05,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/112d9c25340b4686910b8b182fe5b263 2024-11-14T09:44:05,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/a069700ee8cb48699dd6d86166d86bf1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/a069700ee8cb48699dd6d86166d86bf1 2024-11-14T09:44:05,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/a069700ee8cb48699dd6d86166d86bf1, entries=150, sequenceid=60, filesize=30.2 K 2024-11-14T09:44:05,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/658dca2c8a4b48f7b6418d3f5b58d83f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/658dca2c8a4b48f7b6418d3f5b58d83f 2024-11-14T09:44:05,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/658dca2c8a4b48f7b6418d3f5b58d83f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:44:05,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/112d9c25340b4686910b8b182fe5b263 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/112d9c25340b4686910b8b182fe5b263 2024-11-14T09:44:05,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/112d9c25340b4686910b8b182fe5b263, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:44:05,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 16957b2537beefdba492fe73e43233c2 in 140ms, sequenceid=60, compaction requested=true 2024-11-14T09:44:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:05,143 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:05,143 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:05,144 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:05,144 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:05,144 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/B is initiating minor compaction (all files) 2024-11-14T09:44:05,144 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/A is initiating minor compaction (all files) 2024-11-14T09:44:05,144 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/A in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,144 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/B in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,144 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/58518b4d05334d61a2b76b2bb0bb740a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3dde710441074c8fbaf843969aaad4db, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/a069700ee8cb48699dd6d86166d86bf1] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=90.7 K 2024-11-14T09:44:05,144 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/88a1b3abb0104edf9ecebf8bc5fd5e35, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/3169810c41dc49408bb22af78443c130, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/658dca2c8a4b48f7b6418d3f5b58d83f] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=35.2 K 2024-11-14T09:44:05,144 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,145 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/58518b4d05334d61a2b76b2bb0bb740a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3dde710441074c8fbaf843969aaad4db, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/a069700ee8cb48699dd6d86166d86bf1] 2024-11-14T09:44:05,145 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a1b3abb0104edf9ecebf8bc5fd5e35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577443089 2024-11-14T09:44:05,145 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58518b4d05334d61a2b76b2bb0bb740a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577443089 2024-11-14T09:44:05,145 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 3169810c41dc49408bb22af78443c130, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731577443189 2024-11-14T09:44:05,146 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 658dca2c8a4b48f7b6418d3f5b58d83f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577444367 2024-11-14T09:44:05,147 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dde710441074c8fbaf843969aaad4db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731577443189 2024-11-14T09:44:05,147 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting a069700ee8cb48699dd6d86166d86bf1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577444367 2024-11-14T09:44:05,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577445027, max timestamp=1731577445151Num uniques 859; 2024-11-14T09:44:05,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:05,156 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:05,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577445027, max timestamp=1731577445151Num uniques 859; 2024-11-14T09:44:05,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:05,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-14T09:44:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:05,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:05,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:05,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:05,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:05,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:05,165 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:05,166 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0d9d31e620c3411dab12f98458942f6f is 50, key is test_row_0/B:col10/1731577444371/Put/seqid=0 2024-11-14T09:44:05,168 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111439647642028a4617abbb706d633d7eec_16957b2537beefdba492fe73e43233c2 store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:05,174 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111439647642028a4617abbb706d633d7eec_16957b2537beefdba492fe73e43233c2, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:05,174 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111439647642028a4617abbb706d633d7eec_16957b2537beefdba492fe73e43233c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:05,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1f3b9c945e248de9549710987e032fe_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577445027/Put/seqid=0 2024-11-14T09:44:05,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741963_1139 (size=12104) 2024-11-14T09:44:05,198 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0d9d31e620c3411dab12f98458942f6f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0d9d31e620c3411dab12f98458942f6f 2024-11-14T09:44:05,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577505195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577505198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577505200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731577445167, max timestamp=1731577445202Num uniques 859; 2024-11-14T09:44:05,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731577445167, max timestamp=1731577445202Num uniques 859; 2024-11-14T09:44:05,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-14T09:44:05,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-14T09:44:05,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731577445167, max timestamp=1731577445202Num uniques 859; 2024-11-14T09:44:05,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-14T09:44:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,212 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/B of 16957b2537beefdba492fe73e43233c2 into 0d9d31e620c3411dab12f98458942f6f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:05,213 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:05,213 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/B, priority=13, startTime=1731577445143; duration=0sec 2024-11-14T09:44:05,213 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:05,213 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:B 2024-11-14T09:44:05,213 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:05,217 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:05,218 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/C is initiating minor compaction (all files) 2024-11-14T09:44:05,218 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/C in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,218 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/58f399f4c97f4e6284e50403022187d1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/02aa8bb257464a7183a8e74272171f43, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/112d9c25340b4686910b8b182fe5b263] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=35.2 K 2024-11-14T09:44:05,218 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f399f4c97f4e6284e50403022187d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577443089 2024-11-14T09:44:05,219 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 02aa8bb257464a7183a8e74272171f43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731577443189 2024-11-14T09:44:05,219 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 112d9c25340b4686910b8b182fe5b263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577444367 2024-11-14T09:44:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T09:44:05,221 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:05,222 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-14T09:44:05,229 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:05,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741964_1140 (size=4469) 2024-11-14T09:44:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T09:44:05,230 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:05,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:05,232 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#A#compaction#120 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:05,232 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/8bb9e1ce132c4fc69289b358d06fcc81 is 175, key is test_row_0/A:col10/1731577444371/Put/seqid=0 2024-11-14T09:44:05,234 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:05,234 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/b8f105852fec404b92f8726b1a64b75f is 50, key is test_row_0/C:col10/1731577444371/Put/seqid=0 2024-11-14T09:44:05,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741965_1141 (size=14594) 2024-11-14T09:44:05,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577505254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577505257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741966_1142 (size=31058) 2024-11-14T09:44:05,278 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/8bb9e1ce132c4fc69289b358d06fcc81 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/8bb9e1ce132c4fc69289b358d06fcc81 2024-11-14T09:44:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741967_1143 (size=12104) 2024-11-14T09:44:05,285 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/A of 16957b2537beefdba492fe73e43233c2 into 8bb9e1ce132c4fc69289b358d06fcc81(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:05,285 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:05,285 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/A, priority=13, startTime=1731577445143; duration=0sec 2024-11-14T09:44:05,285 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:05,285 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:A 2024-11-14T09:44:05,295 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/b8f105852fec404b92f8726b1a64b75f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b8f105852fec404b92f8726b1a64b75f 2024-11-14T09:44:05,308 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/C of 16957b2537beefdba492fe73e43233c2 into b8f105852fec404b92f8726b1a64b75f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:05,308 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:05,308 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/C, priority=13, startTime=1731577445143; duration=0sec 2024-11-14T09:44:05,308 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:05,308 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:C 2024-11-14T09:44:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577505310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577505310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577505310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T09:44:05,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577505370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577505370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:05,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:05,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577505521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577505521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577505530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:05,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T09:44:05,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577505581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577505581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,637 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:05,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1f3b9c945e248de9549710987e032fe_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1f3b9c945e248de9549710987e032fe_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:05,642 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/492343f4078042dfb2d60038ecde328e, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:05,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/492343f4078042dfb2d60038ecde328e is 175, key is test_row_0/A:col10/1731577445027/Put/seqid=0 2024-11-14T09:44:05,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741968_1144 (size=39549) 2024-11-14T09:44:05,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:05,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577505830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577505830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577505840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:05,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:05,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:05,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T09:44:05,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577505890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577505890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:06,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:06,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/492343f4078042dfb2d60038ecde328e 2024-11-14T09:44:06,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/8ecdd9c143504ae29d627a4555bc667c is 50, key is test_row_0/B:col10/1731577445027/Put/seqid=0 2024-11-14T09:44:06,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741969_1145 (size=12001) 2024-11-14T09:44:06,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/8ecdd9c143504ae29d627a4555bc667c 2024-11-14T09:44:06,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/76368326655c42febc20d9d312e5432a is 50, key is test_row_0/C:col10/1731577445027/Put/seqid=0 2024-11-14T09:44:06,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741970_1146 (size=12001) 2024-11-14T09:44:06,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577506340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577506341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:06,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577506360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T09:44:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577506401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:06,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577506410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:06,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:06,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:06,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/76368326655c42febc20d9d312e5432a 2024-11-14T09:44:06,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/492343f4078042dfb2d60038ecde328e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/492343f4078042dfb2d60038ecde328e 2024-11-14T09:44:06,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/492343f4078042dfb2d60038ecde328e, entries=200, sequenceid=81, filesize=38.6 K 2024-11-14T09:44:06,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/8ecdd9c143504ae29d627a4555bc667c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8ecdd9c143504ae29d627a4555bc667c 2024-11-14T09:44:06,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8ecdd9c143504ae29d627a4555bc667c, entries=150, sequenceid=81, filesize=11.7 K 2024-11-14T09:44:06,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/76368326655c42febc20d9d312e5432a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/76368326655c42febc20d9d312e5432a 2024-11-14T09:44:06,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/76368326655c42febc20d9d312e5432a, entries=150, sequenceid=81, filesize=11.7 K 2024-11-14T09:44:06,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=134.18 KB/137400 for 16957b2537beefdba492fe73e43233c2 in 1354ms, sequenceid=81, compaction requested=false 2024-11-14T09:44:06,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:06,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T09:44:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:06,618 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=134.18 KB heapSize=356.86 KB 2024-11-14T09:44:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:06,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:06,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:06,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:06,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:06,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114485bd38797d34626a0c1c907add32652_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577445250/Put/seqid=0 2024-11-14T09:44:06,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741971_1147 (size=12154) 2024-11-14T09:44:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:07,036 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114485bd38797d34626a0c1c907add32652_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114485bd38797d34626a0c1c907add32652_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/13440659e117492d95b447dfb3e25f65, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:07,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/13440659e117492d95b447dfb3e25f65 is 175, key is test_row_0/A:col10/1731577445250/Put/seqid=0 2024-11-14T09:44:07,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741972_1148 (size=30955) 2024-11-14T09:44:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:07,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:07,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577507371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T09:44:07,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577507379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577507379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577507420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577507420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,422 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,423 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:07,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,455 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/13440659e117492d95b447dfb3e25f65 2024-11-14T09:44:07,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/cdd5e7811eca4f75af75c6b30c64fca7 is 50, key is test_row_0/B:col10/1731577445250/Put/seqid=0 2024-11-14T09:44:07,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741973_1149 (size=12001) 2024-11-14T09:44:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577507490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577507490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577507490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577507700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577507701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577507701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:07,870 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/cdd5e7811eca4f75af75c6b30c64fca7 2024-11-14T09:44:07,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/a108dfdea5754045905de4d42077a5fd is 50, key is test_row_0/C:col10/1731577445250/Put/seqid=0 2024-11-14T09:44:07,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741974_1150 (size=12001) 2024-11-14T09:44:08,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577508011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577508011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577508020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,290 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/a108dfdea5754045905de4d42077a5fd 2024-11-14T09:44:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/13440659e117492d95b447dfb3e25f65 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/13440659e117492d95b447dfb3e25f65 2024-11-14T09:44:08,299 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/13440659e117492d95b447dfb3e25f65, entries=150, sequenceid=115, filesize=30.2 K 2024-11-14T09:44:08,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/cdd5e7811eca4f75af75c6b30c64fca7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/cdd5e7811eca4f75af75c6b30c64fca7 2024-11-14T09:44:08,305 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/cdd5e7811eca4f75af75c6b30c64fca7, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T09:44:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/a108dfdea5754045905de4d42077a5fd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/a108dfdea5754045905de4d42077a5fd 2024-11-14T09:44:08,322 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/a108dfdea5754045905de4d42077a5fd, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T09:44:08,323 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~356.81 KB/365376, currentSize=60.38 KB/61830 for 16957b2537beefdba492fe73e43233c2 in 1705ms, sequenceid=115, compaction requested=true 2024-11-14T09:44:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:08,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-14T09:44:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-14T09:44:08,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-14T09:44:08,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0940 sec 2024-11-14T09:44:08,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.1050 sec 2024-11-14T09:44:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:08,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:44:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:08,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aaa3d57887914dbcb56448c1081a4cf1_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577447367/Put/seqid=0 2024-11-14T09:44:08,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741975_1151 (size=14594) 2024-11-14T09:44:08,548 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:08,558 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114aaa3d57887914dbcb56448c1081a4cf1_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aaa3d57887914dbcb56448c1081a4cf1_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:08,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc8bac464b4f4ba98d7a078443a45763, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:08,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc8bac464b4f4ba98d7a078443a45763 is 175, key is test_row_0/A:col10/1731577447367/Put/seqid=0 2024-11-14T09:44:08,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577508565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577508567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577508568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741976_1152 (size=39549) 2024-11-14T09:44:08,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc8bac464b4f4ba98d7a078443a45763 2024-11-14T09:44:08,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/f1afcd96921141c087bfd0a9996399d1 is 50, key is test_row_0/B:col10/1731577447367/Put/seqid=0 2024-11-14T09:44:08,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741977_1153 (size=12001) 2024-11-14T09:44:08,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/f1afcd96921141c087bfd0a9996399d1 2024-11-14T09:44:08,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/84ae62dbcb3144a6b537ec18be26abab is 50, key is test_row_0/C:col10/1731577447367/Put/seqid=0 2024-11-14T09:44:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741978_1154 (size=12001) 2024-11-14T09:44:08,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/84ae62dbcb3144a6b537ec18be26abab 2024-11-14T09:44:08,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc8bac464b4f4ba98d7a078443a45763 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc8bac464b4f4ba98d7a078443a45763 2024-11-14T09:44:08,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc8bac464b4f4ba98d7a078443a45763, entries=200, sequenceid=128, filesize=38.6 K 2024-11-14T09:44:08,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/f1afcd96921141c087bfd0a9996399d1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f1afcd96921141c087bfd0a9996399d1 2024-11-14T09:44:08,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f1afcd96921141c087bfd0a9996399d1, entries=150, sequenceid=128, filesize=11.7 K 2024-11-14T09:44:08,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/84ae62dbcb3144a6b537ec18be26abab as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84ae62dbcb3144a6b537ec18be26abab 2024-11-14T09:44:08,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84ae62dbcb3144a6b537ec18be26abab, entries=150, sequenceid=128, filesize=11.7 K 2024-11-14T09:44:08,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 16957b2537beefdba492fe73e43233c2 in 137ms, sequenceid=128, compaction requested=true 2024-11-14T09:44:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:08,662 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:08,662 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:08,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:08,664 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:08,664 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:08,664 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/A is initiating minor compaction (all files) 2024-11-14T09:44:08,664 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/B is initiating minor compaction (all files) 2024-11-14T09:44:08,664 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/B in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:08,664 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/A in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:08,664 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0d9d31e620c3411dab12f98458942f6f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8ecdd9c143504ae29d627a4555bc667c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/cdd5e7811eca4f75af75c6b30c64fca7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f1afcd96921141c087bfd0a9996399d1] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=47.0 K 2024-11-14T09:44:08,664 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/8bb9e1ce132c4fc69289b358d06fcc81, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/492343f4078042dfb2d60038ecde328e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/13440659e117492d95b447dfb3e25f65, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc8bac464b4f4ba98d7a078443a45763] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=137.8 K 2024-11-14T09:44:08,664 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:08,664 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/8bb9e1ce132c4fc69289b358d06fcc81, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/492343f4078042dfb2d60038ecde328e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/13440659e117492d95b447dfb3e25f65, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc8bac464b4f4ba98d7a078443a45763] 2024-11-14T09:44:08,664 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9d31e620c3411dab12f98458942f6f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577444367 2024-11-14T09:44:08,665 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb9e1ce132c4fc69289b358d06fcc81, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577444367 2024-11-14T09:44:08,665 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ecdd9c143504ae29d627a4555bc667c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577445027 2024-11-14T09:44:08,665 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492343f4078042dfb2d60038ecde328e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577445027 2024-11-14T09:44:08,665 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd5e7811eca4f75af75c6b30c64fca7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577445250 2024-11-14T09:44:08,667 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13440659e117492d95b447dfb3e25f65, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577445250 2024-11-14T09:44:08,668 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f1afcd96921141c087bfd0a9996399d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731577447367 2024-11-14T09:44:08,668 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc8bac464b4f4ba98d7a078443a45763, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731577447367 2024-11-14T09:44:08,680 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:08,680 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/c01a44db5c2545978b3ab055a8587858 is 50, key is test_row_0/B:col10/1731577447367/Put/seqid=0 2024-11-14T09:44:08,681 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:08,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577448561, max timestamp=1731577448683Num uniques 859; 2024-11-14T09:44:08,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:08,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=147.60 KB heapSize=439.73 KB 2024-11-14T09:44:08,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577448561, max timestamp=1731577448683Num uniques 859; 2024-11-14T09:44:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741979_1155 (size=12241) 2024-11-14T09:44:08,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:08,693 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ed04940de963462594b155819d3b2b6d_16957b2537beefdba492fe73e43233c2 store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:08,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:08,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:08,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:08,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:08,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:08,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:08,699 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ed04940de963462594b155819d3b2b6d_16957b2537beefdba492fe73e43233c2, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:08,699 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ed04940de963462594b155819d3b2b6d_16957b2537beefdba492fe73e43233c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:08,712 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/c01a44db5c2545978b3ab055a8587858 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/c01a44db5c2545978b3ab055a8587858 2024-11-14T09:44:08,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114291cc8bc3b544aabb51e6f264aa2138f_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577448692/Put/seqid=0 2024-11-14T09:44:08,725 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 16957b2537beefdba492fe73e43233c2/B of 16957b2537beefdba492fe73e43233c2 into c01a44db5c2545978b3ab055a8587858(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:08,725 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:08,725 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/B, priority=12, startTime=1731577448662; duration=0sec 2024-11-14T09:44:08,725 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:08,725 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:B 2024-11-14T09:44:08,725 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:08,728 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:08,728 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/C is initiating minor compaction (all files) 2024-11-14T09:44:08,728 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/C in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:08,728 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b8f105852fec404b92f8726b1a64b75f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/76368326655c42febc20d9d312e5432a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/a108dfdea5754045905de4d42077a5fd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84ae62dbcb3144a6b537ec18be26abab] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=47.0 K 2024-11-14T09:44:08,730 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f105852fec404b92f8726b1a64b75f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577444367 2024-11-14T09:44:08,730 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 76368326655c42febc20d9d312e5432a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577445027 2024-11-14T09:44:08,730 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a108dfdea5754045905de4d42077a5fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731577445250 2024-11-14T09:44:08,731 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ae62dbcb3144a6b537ec18be26abab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731577447367 2024-11-14T09:44:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577508726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577508728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577508729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741980_1156 (size=4469) 2024-11-14T09:44:08,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741981_1157 (size=12304) 2024-11-14T09:44:08,768 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:08,769 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#C#compaction#135 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:08,769 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/0de21772cd5c4dc483ad4ead30738a45 is 50, key is test_row_0/C:col10/1731577447367/Put/seqid=0 2024-11-14T09:44:08,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114291cc8bc3b544aabb51e6f264aa2138f_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114291cc8bc3b544aabb51e6f264aa2138f_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:08,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/e32cccc055a14154bfea1ec8b1f71d35, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:08,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/e32cccc055a14154bfea1ec8b1f71d35 is 175, key is test_row_0/A:col10/1731577448692/Put/seqid=0 2024-11-14T09:44:08,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741982_1158 (size=12241) 2024-11-14T09:44:08,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741983_1159 (size=31105) 2024-11-14T09:44:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577508840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577508850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577508850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577509050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577509060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577509060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,167 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#A#compaction#133 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:09,167 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/4f0118c79965498a90472c191efade24 is 175, key is test_row_0/A:col10/1731577447367/Put/seqid=0 2024-11-14T09:44:09,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741984_1160 (size=31195) 2024-11-14T09:44:09,182 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/e32cccc055a14154bfea1ec8b1f71d35 2024-11-14T09:44:09,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/0de21772cd5c4dc483ad4ead30738a45 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/0de21772cd5c4dc483ad4ead30738a45 2024-11-14T09:44:09,194 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 16957b2537beefdba492fe73e43233c2/C of 16957b2537beefdba492fe73e43233c2 into 0de21772cd5c4dc483ad4ead30738a45(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:09,194 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:09,194 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/C, priority=12, startTime=1731577448662; duration=0sec 2024-11-14T09:44:09,194 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:09,194 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:C 2024-11-14T09:44:09,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/9bb59f6bc3684c50bbac9bc70bbbc4b1 is 50, key is test_row_0/B:col10/1731577448692/Put/seqid=0 2024-11-14T09:44:09,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741985_1161 (size=12151) 2024-11-14T09:44:09,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577509360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577509370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577509370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T09:44:09,401 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:09,401 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-14T09:44:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:44:09,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:09,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:09,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577509440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,445 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577509450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:09,454 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:44:09,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:44:09,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:09,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:09,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:09,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:09,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:09,577 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/4f0118c79965498a90472c191efade24 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/4f0118c79965498a90472c191efade24 2024-11-14T09:44:09,589 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 16957b2537beefdba492fe73e43233c2/A of 16957b2537beefdba492fe73e43233c2 into 4f0118c79965498a90472c191efade24(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:09,589 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:09,589 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/A, priority=12, startTime=1731577448662; duration=0sec 2024-11-14T09:44:09,589 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:09,589 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:A 2024-11-14T09:44:09,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/9bb59f6bc3684c50bbac9bc70bbbc4b1 2024-11-14T09:44:09,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c906ae0410214dd58fce09aa922dc58e is 50, key is test_row_0/C:col10/1731577448692/Put/seqid=0 2024-11-14T09:44:09,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741986_1162 (size=12151) 2024-11-14T09:44:09,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c906ae0410214dd58fce09aa922dc58e 2024-11-14T09:44:09,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/e32cccc055a14154bfea1ec8b1f71d35 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e32cccc055a14154bfea1ec8b1f71d35 2024-11-14T09:44:09,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e32cccc055a14154bfea1ec8b1f71d35, entries=150, sequenceid=154, filesize=30.4 K 2024-11-14T09:44:09,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/9bb59f6bc3684c50bbac9bc70bbbc4b1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/9bb59f6bc3684c50bbac9bc70bbbc4b1 2024-11-14T09:44:09,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/9bb59f6bc3684c50bbac9bc70bbbc4b1, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T09:44:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c906ae0410214dd58fce09aa922dc58e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c906ae0410214dd58fce09aa922dc58e 2024-11-14T09:44:09,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c906ae0410214dd58fce09aa922dc58e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T09:44:09,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~273.89 KB/280464, currentSize=80.51 KB/82440 for 16957b2537beefdba492fe73e43233c2 in 971ms, sequenceid=154, compaction requested=false 2024-11-14T09:44:09,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:09,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T09:44:09,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:09,710 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:44:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:44:09,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bd05f70b89a64f5ca55a91dfe0798da4_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577448726/Put/seqid=0 2024-11-14T09:44:09,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741987_1163 (size=12304) 2024-11-14T09:44:09,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,753 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bd05f70b89a64f5ca55a91dfe0798da4_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bd05f70b89a64f5ca55a91dfe0798da4_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:09,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/d243711163fb468f8b4835e1b8e211f7, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:09,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/d243711163fb468f8b4835e1b8e211f7 is 175, key is test_row_0/A:col10/1731577448726/Put/seqid=0 2024-11-14T09:44:09,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741988_1164 (size=31105) 2024-11-14T09:44:09,762 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/d243711163fb468f8b4835e1b8e211f7 2024-11-14T09:44:09,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/2124c51e467546ed9f3466d826aad1ee is 50, key is test_row_0/B:col10/1731577448726/Put/seqid=0 2024-11-14T09:44:09,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741989_1165 (size=12151) 2024-11-14T09:44:09,792 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/2124c51e467546ed9f3466d826aad1ee 2024-11-14T09:44:09,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/e7d071f46cc249d4ad7f0e74bb0222af is 50, key is test_row_0/C:col10/1731577448726/Put/seqid=0 2024-11-14T09:44:09,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741990_1166 (size=12151) 2024-11-14T09:44:09,823 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/e7d071f46cc249d4ad7f0e74bb0222af 2024-11-14T09:44:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/d243711163fb468f8b4835e1b8e211f7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/d243711163fb468f8b4835e1b8e211f7 2024-11-14T09:44:09,841 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/d243711163fb468f8b4835e1b8e211f7, entries=150, sequenceid=172, filesize=30.4 K 2024-11-14T09:44:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/2124c51e467546ed9f3466d826aad1ee as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2124c51e467546ed9f3466d826aad1ee 2024-11-14T09:44:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,850 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2124c51e467546ed9f3466d826aad1ee, entries=150, sequenceid=172, filesize=11.9 K 2024-11-14T09:44:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/e7d071f46cc249d4ad7f0e74bb0222af as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e7d071f46cc249d4ad7f0e74bb0222af 2024-11-14T09:44:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,856 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e7d071f46cc249d4ad7f0e74bb0222af, entries=150, sequenceid=172, filesize=11.9 K 2024-11-14T09:44:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,860 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 16957b2537beefdba492fe73e43233c2 in 150ms, sequenceid=172, compaction requested=true 2024-11-14T09:44:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-14T09:44:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-14T09:44:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T09:44:09,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 459 msec 2024-11-14T09:44:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 464 msec 2024-11-14T09:44:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:09,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:44:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:09,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a3ef05e1073d41718a09b5a8c6df768c_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577449903/Put/seqid=0 2024-11-14T09:44:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:10,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741991_1167 (size=22268) 2024-11-14T09:44:10,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577449985, max timestamp=1731577450002Num uniques 859; 2024-11-14T09:44:10,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577449985, max timestamp=1731577450002Num uniques 859; 2024-11-14T09:44:10,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:10,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:10,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577449985, max timestamp=1731577450002Num uniques 859; 2024-11-14T09:44:10,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T09:44:10,031 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:10,032 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T09:44:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:44:10,034 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:10,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:10,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:10,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731577450041, max timestamp=1731577450060Num uniques 1068; 2024-11-14T09:44:10,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731577450041, max timestamp=1731577450060Num uniques 1068; 2024-11-14T09:44:10,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T09:44:10,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T09:44:10,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731577450041, max timestamp=1731577450060Num uniques 1068; 2024-11-14T09:44:10,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T09:44:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577510125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:44:10,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577510129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577510129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:44:10,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:10,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577510230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577510250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577510251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:44:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:44:10,402 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:10,407 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a3ef05e1073d41718a09b5a8c6df768c_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3ef05e1073d41718a09b5a8c6df768c_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:10,408 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/23668b01432a4894948f82ef6dd974a8, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:10,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/23668b01432a4894948f82ef6dd974a8 is 175, key is test_row_0/A:col10/1731577449903/Put/seqid=0 2024-11-14T09:44:10,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741992_1168 (size=65673) 2024-11-14T09:44:10,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/23668b01432a4894948f82ef6dd974a8 2024-11-14T09:44:10,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577510440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/2370024161854191935e01016fc01952 is 50, key is test_row_0/B:col10/1731577449903/Put/seqid=0 2024-11-14T09:44:10,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741993_1169 (size=12151) 2024-11-14T09:44:10,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577510460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577510460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:44:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:44:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:44:10,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577510750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577510770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577510780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:10,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:44:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:10,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:10,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/2370024161854191935e01016fc01952 2024-11-14T09:44:10,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/806ccc6e90c64cc895cc3af6e8682b7f is 50, key is test_row_0/C:col10/1731577449903/Put/seqid=0 2024-11-14T09:44:10,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741994_1170 (size=12151) 2024-11-14T09:44:10,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/806ccc6e90c64cc895cc3af6e8682b7f 2024-11-14T09:44:10,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/23668b01432a4894948f82ef6dd974a8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/23668b01432a4894948f82ef6dd974a8 2024-11-14T09:44:10,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/23668b01432a4894948f82ef6dd974a8, entries=350, sequenceid=182, filesize=64.1 K 2024-11-14T09:44:10,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/2370024161854191935e01016fc01952 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2370024161854191935e01016fc01952 2024-11-14T09:44:10,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2370024161854191935e01016fc01952, entries=150, sequenceid=182, filesize=11.9 K 2024-11-14T09:44:10,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/806ccc6e90c64cc895cc3af6e8682b7f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/806ccc6e90c64cc895cc3af6e8682b7f 2024-11-14T09:44:10,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/806ccc6e90c64cc895cc3af6e8682b7f, entries=150, sequenceid=182, filesize=11.9 K 2024-11-14T09:44:10,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 16957b2537beefdba492fe73e43233c2 in 971ms, sequenceid=182, compaction requested=true 2024-11-14T09:44:10,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:10,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:10,921 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:10,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:10,922 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:10,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:10,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:10,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:10,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:10,923 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:10,923 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/B is initiating minor compaction (all files) 2024-11-14T09:44:10,923 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/B in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,923 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/c01a44db5c2545978b3ab055a8587858, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/9bb59f6bc3684c50bbac9bc70bbbc4b1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2124c51e467546ed9f3466d826aad1ee, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2370024161854191935e01016fc01952] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=47.6 K 2024-11-14T09:44:10,924 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159078 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:10,924 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/A is initiating minor compaction (all files) 2024-11-14T09:44:10,924 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/A in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,924 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/4f0118c79965498a90472c191efade24, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e32cccc055a14154bfea1ec8b1f71d35, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/d243711163fb468f8b4835e1b8e211f7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/23668b01432a4894948f82ef6dd974a8] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=155.3 K 2024-11-14T09:44:10,924 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c01a44db5c2545978b3ab055a8587858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731577447367 2024-11-14T09:44:10,924 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,924 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/4f0118c79965498a90472c191efade24, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e32cccc055a14154bfea1ec8b1f71d35, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/d243711163fb468f8b4835e1b8e211f7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/23668b01432a4894948f82ef6dd974a8] 2024-11-14T09:44:10,924 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb59f6bc3684c50bbac9bc70bbbc4b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577448561 2024-11-14T09:44:10,924 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f0118c79965498a90472c191efade24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731577447367 2024-11-14T09:44:10,925 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e32cccc055a14154bfea1ec8b1f71d35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577448561 2024-11-14T09:44:10,926 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d243711163fb468f8b4835e1b8e211f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731577448722 2024-11-14T09:44:10,926 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23668b01432a4894948f82ef6dd974a8, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731577449903 2024-11-14T09:44:10,928 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 2124c51e467546ed9f3466d826aad1ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731577448722 2024-11-14T09:44:10,929 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 2370024161854191935e01016fc01952, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731577449903 2024-11-14T09:44:10,946 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:10,947 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:10,948 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/df11dfdf0c7e4f23bcd7d4f532c66b97 is 50, key is test_row_0/B:col10/1731577449903/Put/seqid=0 2024-11-14T09:44:10,949 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a3b924c0aa7d4f7882f1f5a91599f5cd_16957b2537beefdba492fe73e43233c2 store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:10,952 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a3b924c0aa7d4f7882f1f5a91599f5cd_16957b2537beefdba492fe73e43233c2, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:10,952 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a3b924c0aa7d4f7882f1f5a91599f5cd_16957b2537beefdba492fe73e43233c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:10,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:44:10,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:10,956 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=138.65 KB heapSize=389.61 KB 2024-11-14T09:44:10,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:10,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:10,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:10,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:10,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:10,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:10,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741995_1171 (size=12527) 2024-11-14T09:44:10,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741996_1172 (size=4469) 2024-11-14T09:44:10,983 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#A#compaction#145 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:10,984 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/62b8a6d196964c2f848a437d1e988bab is 175, key is test_row_0/A:col10/1731577449903/Put/seqid=0 2024-11-14T09:44:10,991 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/df11dfdf0c7e4f23bcd7d4f532c66b97 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/df11dfdf0c7e4f23bcd7d4f532c66b97 2024-11-14T09:44:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4e34b704ea74b6d8b3c99fedc0c600d_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577450124/Put/seqid=0 2024-11-14T09:44:11,000 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 16957b2537beefdba492fe73e43233c2/B of 16957b2537beefdba492fe73e43233c2 into df11dfdf0c7e4f23bcd7d4f532c66b97(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:11,000 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:11,000 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/B, priority=12, startTime=1731577450921; duration=0sec 2024-11-14T09:44:11,000 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:11,000 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:B 2024-11-14T09:44:11,000 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:11,005 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:11,005 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/C is initiating minor compaction (all files) 2024-11-14T09:44:11,005 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/C in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:11,005 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/0de21772cd5c4dc483ad4ead30738a45, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c906ae0410214dd58fce09aa922dc58e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e7d071f46cc249d4ad7f0e74bb0222af, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/806ccc6e90c64cc895cc3af6e8682b7f] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=47.6 K 2024-11-14T09:44:11,006 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de21772cd5c4dc483ad4ead30738a45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731577447367 2024-11-14T09:44:11,006 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c906ae0410214dd58fce09aa922dc58e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577448561 2024-11-14T09:44:11,006 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e7d071f46cc249d4ad7f0e74bb0222af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731577448722 2024-11-14T09:44:11,007 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 806ccc6e90c64cc895cc3af6e8682b7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731577449903 2024-11-14T09:44:11,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741997_1173 (size=31481) 2024-11-14T09:44:11,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741998_1174 (size=12304) 2024-11-14T09:44:11,018 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:11,019 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/19364593e6ed45c7bf8aec02a84bf123 is 50, key is test_row_0/C:col10/1731577449903/Put/seqid=0 2024-11-14T09:44:11,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741999_1175 (size=12527) 2024-11-14T09:44:11,064 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/19364593e6ed45c7bf8aec02a84bf123 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/19364593e6ed45c7bf8aec02a84bf123 2024-11-14T09:44:11,070 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 16957b2537beefdba492fe73e43233c2/C of 16957b2537beefdba492fe73e43233c2 into 19364593e6ed45c7bf8aec02a84bf123(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:11,070 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:11,070 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/C, priority=12, startTime=1731577450922; duration=0sec 2024-11-14T09:44:11,071 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:11,071 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:C 2024-11-14T09:44:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:44:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:11,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:11,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577511284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577511285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577511300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577511400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577511400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:11,413 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/62b8a6d196964c2f848a437d1e988bab as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/62b8a6d196964c2f848a437d1e988bab 2024-11-14T09:44:11,416 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4e34b704ea74b6d8b3c99fedc0c600d_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e34b704ea74b6d8b3c99fedc0c600d_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:11,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/82976a1a3eb74c20838c29a911a3af37, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:11,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/82976a1a3eb74c20838c29a911a3af37 is 175, key is test_row_0/A:col10/1731577450124/Put/seqid=0 2024-11-14T09:44:11,418 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 16957b2537beefdba492fe73e43233c2/A of 16957b2537beefdba492fe73e43233c2 into 62b8a6d196964c2f848a437d1e988bab(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:11,418 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:11,418 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/A, priority=12, startTime=1731577450921; duration=0sec 2024-11-14T09:44:11,418 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:11,418 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:A 2024-11-14T09:44:11,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742000_1176 (size=31105) 2024-11-14T09:44:11,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577511610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577511610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,833 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/82976a1a3eb74c20838c29a911a3af37 2024-11-14T09:44:11,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/b4bb7216bb5e4091b0d75808c4bf19bd is 50, key is test_row_0/B:col10/1731577450124/Put/seqid=0 2024-11-14T09:44:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742001_1177 (size=12151) 2024-11-14T09:44:11,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577511920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577511920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:44:12,249 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/b4bb7216bb5e4091b0d75808c4bf19bd 2024-11-14T09:44:12,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/5c2b76545505462f990f82d6ebe81890 is 50, key is test_row_0/C:col10/1731577450124/Put/seqid=0 2024-11-14T09:44:12,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742002_1178 (size=12151) 2024-11-14T09:44:12,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577512310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:12,313 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577512430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577512430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:12,662 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/5c2b76545505462f990f82d6ebe81890 2024-11-14T09:44:12,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/82976a1a3eb74c20838c29a911a3af37 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/82976a1a3eb74c20838c29a911a3af37 2024-11-14T09:44:12,671 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/82976a1a3eb74c20838c29a911a3af37, entries=150, sequenceid=235, filesize=30.4 K 2024-11-14T09:44:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/b4bb7216bb5e4091b0d75808c4bf19bd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/b4bb7216bb5e4091b0d75808c4bf19bd 2024-11-14T09:44:12,676 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/b4bb7216bb5e4091b0d75808c4bf19bd, entries=150, sequenceid=235, filesize=11.9 K 2024-11-14T09:44:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/5c2b76545505462f990f82d6ebe81890 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/5c2b76545505462f990f82d6ebe81890 2024-11-14T09:44:12,682 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/5c2b76545505462f990f82d6ebe81890, entries=150, sequenceid=235, filesize=11.9 K 2024-11-14T09:44:12,684 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~389.56 KB/398912, currentSize=46.96 KB/48090 for 16957b2537beefdba492fe73e43233c2 in 1728ms, sequenceid=235, compaction requested=false 2024-11-14T09:44:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T09:44:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T09:44:12,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T09:44:12,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6510 sec 2024-11-14T09:44:12,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.6560 sec 2024-11-14T09:44:12,767 INFO [master/defc576eb6b7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T09:44:12,767 INFO [master/defc576eb6b7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T09:44:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:13,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:44:13,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:13,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:13,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:13,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:13,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:13,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111474ee3aacc95f4b6da404e909f00e8321_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577453442/Put/seqid=0 2024-11-14T09:44:13,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742003_1179 (size=14794) 2024-11-14T09:44:13,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:13,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111474ee3aacc95f4b6da404e909f00e8321_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111474ee3aacc95f4b6da404e909f00e8321_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:13,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc976e0fbf9447ef9d082abdeedab478, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:13,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc976e0fbf9447ef9d082abdeedab478 is 175, key is test_row_0/A:col10/1731577453442/Put/seqid=0 2024-11-14T09:44:13,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742004_1180 (size=39749) 2024-11-14T09:44:13,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577513479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577453452, max timestamp=1731577453479Num uniques 859; 2024-11-14T09:44:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577513480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:44:13,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577453452, max timestamp=1731577453479Num uniques 859; 2024-11-14T09:44:13,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:44:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577453452, max timestamp=1731577453479Num uniques 859; 2024-11-14T09:44:13,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:44:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577513535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577513536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577513590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577513590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577513650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577513650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577513800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577513800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577513861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577513861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:13,877 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc976e0fbf9447ef9d082abdeedab478 2024-11-14T09:44:13,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/6558076f69074356b7ff11dfaf28eacf is 50, key is test_row_0/B:col10/1731577453442/Put/seqid=0 2024-11-14T09:44:13,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742005_1181 (size=12151) 2024-11-14T09:44:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577514110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577514120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577514181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577514181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:44:14,200 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:14,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T09:44:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:44:14,206 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:14,207 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:14,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:14,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/6558076f69074356b7ff11dfaf28eacf 2024-11-14T09:44:14,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c94254dcb579409093d764e27998d14b is 50, key is test_row_0/C:col10/1731577453442/Put/seqid=0 2024-11-14T09:44:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742006_1182 (size=12151) 2024-11-14T09:44:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:44:14,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577514340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,343 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:44:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:44:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:44:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577514621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577514640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:44:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577514690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c94254dcb579409093d764e27998d14b 2024-11-14T09:44:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577514701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/dc976e0fbf9447ef9d082abdeedab478 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc976e0fbf9447ef9d082abdeedab478 2024-11-14T09:44:14,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc976e0fbf9447ef9d082abdeedab478, entries=200, sequenceid=249, filesize=38.8 K 2024-11-14T09:44:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/6558076f69074356b7ff11dfaf28eacf as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/6558076f69074356b7ff11dfaf28eacf 2024-11-14T09:44:14,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/6558076f69074356b7ff11dfaf28eacf, entries=150, sequenceid=249, filesize=11.9 K 2024-11-14T09:44:14,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c94254dcb579409093d764e27998d14b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c94254dcb579409093d764e27998d14b 2024-11-14T09:44:14,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c94254dcb579409093d764e27998d14b, entries=150, sequenceid=249, filesize=11.9 K 2024-11-14T09:44:14,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 16957b2537beefdba492fe73e43233c2 in 1281ms, sequenceid=249, compaction requested=true 2024-11-14T09:44:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:14,724 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:14,724 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:14,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:14,725 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:14,725 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:14,725 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/B is initiating minor compaction (all files) 2024-11-14T09:44:14,725 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/A is initiating minor compaction (all files) 2024-11-14T09:44:14,725 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/B in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,725 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/A in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,725 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/df11dfdf0c7e4f23bcd7d4f532c66b97, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/b4bb7216bb5e4091b0d75808c4bf19bd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/6558076f69074356b7ff11dfaf28eacf] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.0 K 2024-11-14T09:44:14,725 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/62b8a6d196964c2f848a437d1e988bab, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/82976a1a3eb74c20838c29a911a3af37, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc976e0fbf9447ef9d082abdeedab478] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=99.9 K 2024-11-14T09:44:14,725 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,726 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/62b8a6d196964c2f848a437d1e988bab, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/82976a1a3eb74c20838c29a911a3af37, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc976e0fbf9447ef9d082abdeedab478] 2024-11-14T09:44:14,726 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting df11dfdf0c7e4f23bcd7d4f532c66b97, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731577449903 2024-11-14T09:44:14,726 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b8a6d196964c2f848a437d1e988bab, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731577449903 2024-11-14T09:44:14,726 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b4bb7216bb5e4091b0d75808c4bf19bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731577450117 2024-11-14T09:44:14,726 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82976a1a3eb74c20838c29a911a3af37, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731577450117 2024-11-14T09:44:14,727 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 6558076f69074356b7ff11dfaf28eacf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731577451278 2024-11-14T09:44:14,727 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc976e0fbf9447ef9d082abdeedab478, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731577451269 2024-11-14T09:44:14,732 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:14,736 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:14,736 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/645276ba3e8848fdb789607a2e15ba18 is 50, key is test_row_0/B:col10/1731577453442/Put/seqid=0 2024-11-14T09:44:14,736 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114cc5f0804106149ab8ecb987a68ce263e_16957b2537beefdba492fe73e43233c2 store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:14,739 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114cc5f0804106149ab8ecb987a68ce263e_16957b2537beefdba492fe73e43233c2, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:14,739 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cc5f0804106149ab8ecb987a68ce263e_16957b2537beefdba492fe73e43233c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:14,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742008_1184 (size=4469) 2024-11-14T09:44:14,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742007_1183 (size=12629) 2024-11-14T09:44:14,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:44:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:14,819 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T09:44:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:44:14,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146ac5333cc2994dd396ef1d40aab0849f_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577453523/Put/seqid=0 2024-11-14T09:44:14,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742009_1185 (size=12454) 2024-11-14T09:44:15,155 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:15,156 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/e430bead306a4f3b8c11e4ce4ba40a57 is 175, key is test_row_0/A:col10/1731577453442/Put/seqid=0 2024-11-14T09:44:15,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742010_1186 (size=31583) 2024-11-14T09:44:15,169 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/645276ba3e8848fdb789607a2e15ba18 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/645276ba3e8848fdb789607a2e15ba18 2024-11-14T09:44:15,174 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/B of 16957b2537beefdba492fe73e43233c2 into 645276ba3e8848fdb789607a2e15ba18(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:15,174 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:15,174 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/B, priority=13, startTime=1731577454724; duration=0sec 2024-11-14T09:44:15,174 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:15,174 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:B 2024-11-14T09:44:15,174 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:15,175 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:15,176 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/C is initiating minor compaction (all files) 2024-11-14T09:44:15,176 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/C in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:15,176 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/19364593e6ed45c7bf8aec02a84bf123, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/5c2b76545505462f990f82d6ebe81890, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c94254dcb579409093d764e27998d14b] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.0 K 2024-11-14T09:44:15,176 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 19364593e6ed45c7bf8aec02a84bf123, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731577449903 2024-11-14T09:44:15,176 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2b76545505462f990f82d6ebe81890, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731577450117 2024-11-14T09:44:15,177 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c94254dcb579409093d764e27998d14b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731577451278 2024-11-14T09:44:15,184 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:15,184 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/188be3502d0f4dfe8b1d448df94d1822 is 50, key is test_row_0/C:col10/1731577453442/Put/seqid=0 2024-11-14T09:44:15,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742011_1187 (size=12629) 2024-11-14T09:44:15,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:15,241 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146ac5333cc2994dd396ef1d40aab0849f_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146ac5333cc2994dd396ef1d40aab0849f_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/1f07f14976e946c485ccb17a82336397, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/1f07f14976e946c485ccb17a82336397 is 175, key is test_row_0/A:col10/1731577453523/Put/seqid=0 2024-11-14T09:44:15,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742012_1188 (size=31255) 2024-11-14T09:44:15,247 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/1f07f14976e946c485ccb17a82336397 2024-11-14T09:44:15,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0c9985543aa140fbbc43ed63644ce04c is 50, key is test_row_0/B:col10/1731577453523/Put/seqid=0 2024-11-14T09:44:15,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742013_1189 (size=12301) 2024-11-14T09:44:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:44:15,565 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/e430bead306a4f3b8c11e4ce4ba40a57 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e430bead306a4f3b8c11e4ce4ba40a57 2024-11-14T09:44:15,570 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/A of 16957b2537beefdba492fe73e43233c2 into e430bead306a4f3b8c11e4ce4ba40a57(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:15,570 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:15,570 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/A, priority=13, startTime=1731577454724; duration=0sec 2024-11-14T09:44:15,570 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:15,570 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:A 2024-11-14T09:44:15,593 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/188be3502d0f4dfe8b1d448df94d1822 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/188be3502d0f4dfe8b1d448df94d1822 2024-11-14T09:44:15,598 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/C of 16957b2537beefdba492fe73e43233c2 into 188be3502d0f4dfe8b1d448df94d1822(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:15,598 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:15,598 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/C, priority=13, startTime=1731577454724; duration=0sec 2024-11-14T09:44:15,598 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:15,598 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:C 2024-11-14T09:44:15,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:15,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577515651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:15,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577515655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:15,661 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0c9985543aa140fbbc43ed63644ce04c 2024-11-14T09:44:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/4b2e2536aeb94670b87098c8a3fa481b is 50, key is test_row_0/C:col10/1731577453523/Put/seqid=0 2024-11-14T09:44:15,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742014_1190 (size=12301) 2024-11-14T09:44:15,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577515700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,704 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:15,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577515720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,730 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:15,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577515760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577515760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:15,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577515971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577515971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:16,079 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/4b2e2536aeb94670b87098c8a3fa481b 2024-11-14T09:44:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/1f07f14976e946c485ccb17a82336397 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/1f07f14976e946c485ccb17a82336397 2024-11-14T09:44:16,118 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/1f07f14976e946c485ccb17a82336397, entries=150, sequenceid=284, filesize=30.5 K 2024-11-14T09:44:16,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0c9985543aa140fbbc43ed63644ce04c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0c9985543aa140fbbc43ed63644ce04c 2024-11-14T09:44:16,122 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0c9985543aa140fbbc43ed63644ce04c, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T09:44:16,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/4b2e2536aeb94670b87098c8a3fa481b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4b2e2536aeb94670b87098c8a3fa481b 2024-11-14T09:44:16,127 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4b2e2536aeb94670b87098c8a3fa481b, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T09:44:16,128 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 16957b2537beefdba492fe73e43233c2 in 1308ms, sequenceid=284, compaction requested=false 2024-11-14T09:44:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T09:44:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T09:44:16,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T09:44:16,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9220 sec 2024-11-14T09:44:16,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9300 sec 2024-11-14T09:44:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:16,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:44:16,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:16,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:16,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:16,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:16,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:16,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:16,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145295a16566d04c81a5483691dc14dd3d_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577456282/Put/seqid=0 2024-11-14T09:44:16,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742015_1191 (size=14994) 2024-11-14T09:44:16,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577456302, max timestamp=1731577456323Num uniques 859; 2024-11-14T09:44:16,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577456302, max timestamp=1731577456323Num uniques 859; 2024-11-14T09:44:16,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577456302, max timestamp=1731577456323Num uniques 859; 2024-11-14T09:44:16,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:44:16,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:44:16,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:44:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:44:16,350 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:16,351 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T09:44:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:44:16,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:16,353 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:16,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577516356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577516360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:44:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577516470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:16,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577516470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:16,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:44:16,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577516680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577516690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:16,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:16,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145295a16566d04c81a5483691dc14dd3d_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145295a16566d04c81a5483691dc14dd3d_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:16,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/cb1b9f1728404ac4bfc4923a4a0aa2b1, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:16,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/cb1b9f1728404ac4bfc4923a4a0aa2b1 is 175, key is test_row_0/A:col10/1731577456282/Put/seqid=0 2024-11-14T09:44:16,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742016_1192 (size=39949) 2024-11-14T09:44:16,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:16,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:16,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:44:16,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577516990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577517000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:17,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,122 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/cb1b9f1728404ac4bfc4923a4a0aa2b1 2024-11-14T09:44:17,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/f240faa7cb2748e480d0519009eee415 is 50, key is test_row_0/B:col10/1731577456282/Put/seqid=0 2024-11-14T09:44:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742017_1193 (size=12301) 2024-11-14T09:44:17,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:44:17,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577517500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:17,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577517520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:17,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/f240faa7cb2748e480d0519009eee415 2024-11-14T09:44:17,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/7c687b2972234c2fb24ec7852b2572de is 50, key is test_row_0/C:col10/1731577456282/Put/seqid=0 2024-11-14T09:44:17,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742018_1194 (size=12301) 2024-11-14T09:44:17,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/7c687b2972234c2fb24ec7852b2572de 2024-11-14T09:44:17,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/cb1b9f1728404ac4bfc4923a4a0aa2b1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/cb1b9f1728404ac4bfc4923a4a0aa2b1 2024-11-14T09:44:17,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/cb1b9f1728404ac4bfc4923a4a0aa2b1, entries=200, sequenceid=298, filesize=39.0 K 2024-11-14T09:44:17,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/f240faa7cb2748e480d0519009eee415 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f240faa7cb2748e480d0519009eee415 2024-11-14T09:44:17,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f240faa7cb2748e480d0519009eee415, entries=150, sequenceid=298, filesize=12.0 K 2024-11-14T09:44:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/7c687b2972234c2fb24ec7852b2572de as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/7c687b2972234c2fb24ec7852b2572de 2024-11-14T09:44:17,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/7c687b2972234c2fb24ec7852b2572de, entries=150, sequenceid=298, filesize=12.0 K 2024-11-14T09:44:17,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 16957b2537beefdba492fe73e43233c2 in 1281ms, sequenceid=298, compaction requested=true 2024-11-14T09:44:17,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:17,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:17,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:17,564 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:17,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:17,565 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:17,566 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:17,566 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:17,566 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/B is initiating minor compaction (all files) 2024-11-14T09:44:17,566 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/A is initiating minor compaction (all files) 2024-11-14T09:44:17,566 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/B in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,566 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/A in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,566 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/645276ba3e8848fdb789607a2e15ba18, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0c9985543aa140fbbc43ed63644ce04c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f240faa7cb2748e480d0519009eee415] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.4 K 2024-11-14T09:44:17,566 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e430bead306a4f3b8c11e4ce4ba40a57, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/1f07f14976e946c485ccb17a82336397, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/cb1b9f1728404ac4bfc4923a4a0aa2b1] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=100.4 K 2024-11-14T09:44:17,566 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,566 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e430bead306a4f3b8c11e4ce4ba40a57, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/1f07f14976e946c485ccb17a82336397, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/cb1b9f1728404ac4bfc4923a4a0aa2b1] 2024-11-14T09:44:17,566 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 645276ba3e8848fdb789607a2e15ba18, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731577451278 2024-11-14T09:44:17,566 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e430bead306a4f3b8c11e4ce4ba40a57, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731577451278 2024-11-14T09:44:17,567 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9985543aa140fbbc43ed63644ce04c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731577453522 2024-11-14T09:44:17,567 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f07f14976e946c485ccb17a82336397, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731577453522 2024-11-14T09:44:17,567 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1b9f1728404ac4bfc4923a4a0aa2b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731577455643 2024-11-14T09:44:17,567 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f240faa7cb2748e480d0519009eee415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731577455643 2024-11-14T09:44:17,572 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:17,574 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:17,575 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/d0ac4342b3f74b25bd070eae3315e2b4 is 50, key is test_row_0/B:col10/1731577456282/Put/seqid=0 2024-11-14T09:44:17,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:44:17,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:17,577 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-14T09:44:17,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:17,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:17,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:17,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:17,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:17,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:17,584 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114661259cd3e6b478baa98c0cf1c223cb2_16957b2537beefdba492fe73e43233c2 store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:17,587 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114661259cd3e6b478baa98c0cf1c223cb2_16957b2537beefdba492fe73e43233c2, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:17,587 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114661259cd3e6b478baa98c0cf1c223cb2_16957b2537beefdba492fe73e43233c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742019_1195 (size=12881) 2024-11-14T09:44:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742020_1196 (size=4469) 2024-11-14T09:44:17,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111440970b7ca95d41898f6049045c7454d9_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577456341/Put/seqid=0 2024-11-14T09:44:17,616 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#A#compaction#162 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:17,616 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/2d8bd5a12ba641029455571cdb0efb03 is 175, key is test_row_0/A:col10/1731577456282/Put/seqid=0 2024-11-14T09:44:17,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742021_1197 (size=12454) 2024-11-14T09:44:17,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742022_1198 (size=31835) 2024-11-14T09:44:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:17,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:17,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577517766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:17,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577517767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:17,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577517880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577517880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/d0ac4342b3f74b25bd070eae3315e2b4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d0ac4342b3f74b25bd070eae3315e2b4 2024-11-14T09:44:18,008 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/B of 16957b2537beefdba492fe73e43233c2 into d0ac4342b3f74b25bd070eae3315e2b4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:18,008 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:18,008 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/B, priority=13, startTime=1731577457564; duration=0sec 2024-11-14T09:44:18,008 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:18,008 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:B 2024-11-14T09:44:18,008 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:18,009 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:18,009 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/C is initiating minor compaction (all files) 2024-11-14T09:44:18,009 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/C in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:18,010 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/188be3502d0f4dfe8b1d448df94d1822, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4b2e2536aeb94670b87098c8a3fa481b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/7c687b2972234c2fb24ec7852b2572de] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.4 K 2024-11-14T09:44:18,010 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 188be3502d0f4dfe8b1d448df94d1822, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731577451278 2024-11-14T09:44:18,010 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b2e2536aeb94670b87098c8a3fa481b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731577453522 2024-11-14T09:44:18,010 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c687b2972234c2fb24ec7852b2572de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731577455643 2024-11-14T09:44:18,017 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:18,018 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c607d986469a42ca8a865fa89ccae990 is 50, key is test_row_0/C:col10/1731577456282/Put/seqid=0 2024-11-14T09:44:18,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742023_1199 (size=12881) 2024-11-14T09:44:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:18,033 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111440970b7ca95d41898f6049045c7454d9_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111440970b7ca95d41898f6049045c7454d9_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:18,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/5d38a93adc96499185e2314ec9339b8a, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:18,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/5d38a93adc96499185e2314ec9339b8a is 175, key is test_row_0/A:col10/1731577456341/Put/seqid=0 2024-11-14T09:44:18,040 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/2d8bd5a12ba641029455571cdb0efb03 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/2d8bd5a12ba641029455571cdb0efb03 2024-11-14T09:44:18,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742024_1200 (size=31255) 2024-11-14T09:44:18,045 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/A of 16957b2537beefdba492fe73e43233c2 into 2d8bd5a12ba641029455571cdb0efb03(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:18,045 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:18,045 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/A, priority=13, startTime=1731577457564; duration=0sec 2024-11-14T09:44:18,045 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:18,045 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:A 2024-11-14T09:44:18,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577518090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577518091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50474 deadline: 1731577518380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,385 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577518400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577518400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,426 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/c607d986469a42ca8a865fa89ccae990 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c607d986469a42ca8a865fa89ccae990 2024-11-14T09:44:18,430 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/C of 16957b2537beefdba492fe73e43233c2 into c607d986469a42ca8a865fa89ccae990(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:18,430 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:18,430 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/C, priority=13, startTime=1731577457565; duration=0sec 2024-11-14T09:44:18,431 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:18,431 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:C 2024-11-14T09:44:18,441 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/5d38a93adc96499185e2314ec9339b8a 2024-11-14T09:44:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0b8b5bb759064c07b16389bb12ca393e is 50, key is test_row_0/B:col10/1731577456341/Put/seqid=0 2024-11-14T09:44:18,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742025_1201 (size=12301) 2024-11-14T09:44:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:44:18,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577518520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,523 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:18,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577518540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,545 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,858 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0b8b5bb759064c07b16389bb12ca393e 2024-11-14T09:44:18,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/61e9e0be696d4c2b84e9a4a75c34b42e is 50, key is test_row_0/C:col10/1731577456341/Put/seqid=0 2024-11-14T09:44:18,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742026_1202 (size=12301) 2024-11-14T09:44:18,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577518910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577518911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:19,273 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/61e9e0be696d4c2b84e9a4a75c34b42e 2024-11-14T09:44:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/5d38a93adc96499185e2314ec9339b8a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/5d38a93adc96499185e2314ec9339b8a 2024-11-14T09:44:19,281 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/5d38a93adc96499185e2314ec9339b8a, entries=150, sequenceid=331, filesize=30.5 K 2024-11-14T09:44:19,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/0b8b5bb759064c07b16389bb12ca393e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0b8b5bb759064c07b16389bb12ca393e 2024-11-14T09:44:19,287 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0b8b5bb759064c07b16389bb12ca393e, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T09:44:19,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/61e9e0be696d4c2b84e9a4a75c34b42e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/61e9e0be696d4c2b84e9a4a75c34b42e 2024-11-14T09:44:19,306 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/61e9e0be696d4c2b84e9a4a75c34b42e, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T09:44:19,307 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 16957b2537beefdba492fe73e43233c2 in 1731ms, sequenceid=331, compaction requested=false 2024-11-14T09:44:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T09:44:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T09:44:19,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T09:44:19,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9560 sec 2024-11-14T09:44:19,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.9600 sec 2024-11-14T09:44:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:19,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:44:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:19,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b6096d126f3459fb19f43f087eb1b2a_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577459930/Put/seqid=0 2024-11-14T09:44:19,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742027_1203 (size=17534) 2024-11-14T09:44:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577519965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:19,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577519967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577520070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577520080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577520281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577520290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:20,349 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b6096d126f3459fb19f43f087eb1b2a_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b6096d126f3459fb19f43f087eb1b2a_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:20,350 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3d1a0c97bb3c41dcbfdde0899355b354, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:20,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3d1a0c97bb3c41dcbfdde0899355b354 is 175, key is test_row_0/A:col10/1731577459930/Put/seqid=0 2024-11-14T09:44:20,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742028_1204 (size=48639) 2024-11-14T09:44:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:44:20,521 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:20,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T09:44:20,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:44:20,525 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:20,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:20,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50510 deadline: 1731577520550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,553 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50526 deadline: 1731577520570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,571 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 16957b2537beefdba492fe73e43233c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577520591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577520600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:44:20,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:20,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:20,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:20,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:20,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,755 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3d1a0c97bb3c41dcbfdde0899355b354 2024-11-14T09:44:20,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/d1f39508e4ae455a8e15ddbec5695837 is 50, key is test_row_0/B:col10/1731577459930/Put/seqid=0 2024-11-14T09:44:20,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742029_1205 (size=12301) 2024-11-14T09:44:20,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:20,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:44:20,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:20,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:20,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:20,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577521100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:21,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577521110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:21,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:21,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:44:21,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/d1f39508e4ae455a8e15ddbec5695837 2024-11-14T09:44:21,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/4741bc71e7894b51afc5bb97f14dff5b is 50, key is test_row_0/C:col10/1731577459930/Put/seqid=0 2024-11-14T09:44:21,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742030_1206 (size=12301) 2024-11-14T09:44:21,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:21,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:21,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/4741bc71e7894b51afc5bb97f14dff5b 2024-11-14T09:44:21,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/3d1a0c97bb3c41dcbfdde0899355b354 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3d1a0c97bb3c41dcbfdde0899355b354 2024-11-14T09:44:21,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:21,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:21,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3d1a0c97bb3c41dcbfdde0899355b354, entries=250, sequenceid=347, filesize=47.5 K 2024-11-14T09:44:21,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:21,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/d1f39508e4ae455a8e15ddbec5695837 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d1f39508e4ae455a8e15ddbec5695837 2024-11-14T09:44:21,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d1f39508e4ae455a8e15ddbec5695837, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T09:44:21,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/4741bc71e7894b51afc5bb97f14dff5b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4741bc71e7894b51afc5bb97f14dff5b 2024-11-14T09:44:21,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4741bc71e7894b51afc5bb97f14dff5b, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T09:44:21,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 16957b2537beefdba492fe73e43233c2 in 1677ms, sequenceid=347, compaction requested=true 2024-11-14T09:44:21,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:21,608 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:21,609 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:21,610 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:21,610 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/A is initiating minor compaction (all files) 2024-11-14T09:44:21,610 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/A in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,610 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/2d8bd5a12ba641029455571cdb0efb03, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/5d38a93adc96499185e2314ec9339b8a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3d1a0c97bb3c41dcbfdde0899355b354] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=109.1 K 2024-11-14T09:44:21,610 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,610 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/2d8bd5a12ba641029455571cdb0efb03, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/5d38a93adc96499185e2314ec9339b8a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3d1a0c97bb3c41dcbfdde0899355b354] 2024-11-14T09:44:21,610 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:21,610 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/B is initiating minor compaction (all files) 2024-11-14T09:44:21,610 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/B in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,610 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d0ac4342b3f74b25bd070eae3315e2b4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0b8b5bb759064c07b16389bb12ca393e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d1f39508e4ae455a8e15ddbec5695837] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.6 K 2024-11-14T09:44:21,611 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d8bd5a12ba641029455571cdb0efb03, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731577455643 2024-11-14T09:44:21,611 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ac4342b3f74b25bd070eae3315e2b4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731577455643 2024-11-14T09:44:21,611 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d38a93adc96499185e2314ec9339b8a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731577456340 2024-11-14T09:44:21,612 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b8b5bb759064c07b16389bb12ca393e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731577456340 2024-11-14T09:44:21,612 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1a0c97bb3c41dcbfdde0899355b354, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731577457741 2024-11-14T09:44:21,612 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f39508e4ae455a8e15ddbec5695837, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731577457741 2024-11-14T09:44:21,620 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:21,621 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:21,622 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/8c67d6f8887b493ab9ec9a0d209e9768 is 50, key is test_row_0/B:col10/1731577459930/Put/seqid=0 2024-11-14T09:44:21,633 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114214b7ea10a9242b79691ec1d0b410e19_16957b2537beefdba492fe73e43233c2 store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:21,635 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114214b7ea10a9242b79691ec1d0b410e19_16957b2537beefdba492fe73e43233c2, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:21,635 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114214b7ea10a9242b79691ec1d0b410e19_16957b2537beefdba492fe73e43233c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742031_1207 (size=12983) 2024-11-14T09:44:21,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742032_1208 (size=4469) 2024-11-14T09:44:21,649 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#A#compaction#172 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:21,649 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/b141daab0d9a4c4f9f0128bc859658f3 is 175, key is test_row_0/A:col10/1731577459930/Put/seqid=0 2024-11-14T09:44:21,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742033_1209 (size=31937) 2024-11-14T09:44:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:44:21,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:44:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:21,750 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:44:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114adf55cbbd36848daa9caf3d2a0fa15f9_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577459964/Put/seqid=0 2024-11-14T09:44:21,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742034_1210 (size=12454) 2024-11-14T09:44:22,048 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/8c67d6f8887b493ab9ec9a0d209e9768 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8c67d6f8887b493ab9ec9a0d209e9768 2024-11-14T09:44:22,055 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/B of 16957b2537beefdba492fe73e43233c2 into 8c67d6f8887b493ab9ec9a0d209e9768(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:22,055 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:22,055 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/B, priority=13, startTime=1731577461609; duration=0sec 2024-11-14T09:44:22,055 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:22,055 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:B 2024-11-14T09:44:22,055 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:22,057 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:22,057 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/C is initiating minor compaction (all files) 2024-11-14T09:44:22,057 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/C in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:22,058 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c607d986469a42ca8a865fa89ccae990, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/61e9e0be696d4c2b84e9a4a75c34b42e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4741bc71e7894b51afc5bb97f14dff5b] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.6 K 2024-11-14T09:44:22,058 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/b141daab0d9a4c4f9f0128bc859658f3 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/b141daab0d9a4c4f9f0128bc859658f3 2024-11-14T09:44:22,058 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c607d986469a42ca8a865fa89ccae990, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731577455643 2024-11-14T09:44:22,058 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e9e0be696d4c2b84e9a4a75c34b42e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731577456340 2024-11-14T09:44:22,059 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4741bc71e7894b51afc5bb97f14dff5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731577457741 2024-11-14T09:44:22,063 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/A of 16957b2537beefdba492fe73e43233c2 into b141daab0d9a4c4f9f0128bc859658f3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:22,063 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:22,063 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/A, priority=13, startTime=1731577461608; duration=0sec 2024-11-14T09:44:22,063 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:22,063 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:A 2024-11-14T09:44:22,067 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:22,068 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/b69aedd2c5144fa3a540d415cc66c154 is 50, key is test_row_0/C:col10/1731577459930/Put/seqid=0 2024-11-14T09:44:22,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742035_1211 (size=12983) 2024-11-14T09:44:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. as already flushing 2024-11-14T09:44:22,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577522137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577522138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:22,165 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114adf55cbbd36848daa9caf3d2a0fa15f9_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114adf55cbbd36848daa9caf3d2a0fa15f9_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/ba560049145b4636abd708e1b84a3834, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/ba560049145b4636abd708e1b84a3834 is 175, key is test_row_0/A:col10/1731577459964/Put/seqid=0 2024-11-14T09:44:22,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742036_1212 (size=31255) 2024-11-14T09:44:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577522251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577522251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577522470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577522470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,484 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/b69aedd2c5144fa3a540d415cc66c154 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b69aedd2c5144fa3a540d415cc66c154 2024-11-14T09:44:22,489 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/C of 16957b2537beefdba492fe73e43233c2 into b69aedd2c5144fa3a540d415cc66c154(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:22,489 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:22,489 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/C, priority=13, startTime=1731577461609; duration=0sec 2024-11-14T09:44:22,489 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:22,489 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:C 2024-11-14T09:44:22,571 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/ba560049145b4636abd708e1b84a3834 2024-11-14T09:44:22,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/5a04eb65a423451688d303a96b658a7d is 50, key is test_row_0/B:col10/1731577459964/Put/seqid=0 2024-11-14T09:44:22,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742037_1213 (size=12301) 2024-11-14T09:44:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:44:22,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577522780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577522780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:22,983 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/5a04eb65a423451688d303a96b658a7d 2024-11-14T09:44:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/393ce7506dea4c1b98a3044804e4b424 is 50, key is test_row_0/C:col10/1731577459964/Put/seqid=0 2024-11-14T09:44:22,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742038_1214 (size=12301) 2024-11-14T09:44:23,077 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-14T09:44:23,077 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:23,077 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,077 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:23,078 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-14T09:44:23,078 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:23,078 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,078 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-14T09:44:23,078 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:23,078 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,078 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:23,078 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:23,080 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-14T09:44:23,080 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:23,080 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,080 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:23,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:23,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50458 deadline: 1731577523291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:23,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50540 deadline: 1731577523291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:44:23,396 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/393ce7506dea4c1b98a3044804e4b424 2024-11-14T09:44:23,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/ba560049145b4636abd708e1b84a3834 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/ba560049145b4636abd708e1b84a3834 2024-11-14T09:44:23,414 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/ba560049145b4636abd708e1b84a3834, entries=150, sequenceid=366, filesize=30.5 K 2024-11-14T09:44:23,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/5a04eb65a423451688d303a96b658a7d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/5a04eb65a423451688d303a96b658a7d 2024-11-14T09:44:23,419 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/5a04eb65a423451688d303a96b658a7d, entries=150, sequenceid=366, filesize=12.0 K 2024-11-14T09:44:23,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/393ce7506dea4c1b98a3044804e4b424 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/393ce7506dea4c1b98a3044804e4b424 2024-11-14T09:44:23,424 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/393ce7506dea4c1b98a3044804e4b424, entries=150, sequenceid=366, filesize=12.0 K 2024-11-14T09:44:23,425 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 16957b2537beefdba492fe73e43233c2 in 1675ms, sequenceid=366, compaction requested=false 2024-11-14T09:44:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T09:44:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T09:44:23,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T09:44:23,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9010 sec 2024-11-14T09:44:23,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.9070 sec 2024-11-14T09:44:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:24,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:44:24,304 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-14T09:44:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:24,304 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:24,304 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:24,304 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:24,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:24,312 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-14T09:44:24,312 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:24,312 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,312 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:24,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111403938def6520490bb2f84add3fb2d21f_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577462128/Put/seqid=0 2024-11-14T09:44:24,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742039_1215 (size=12454) 2024-11-14T09:44:24,575 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-14T09:44:24,575 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:24,575 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,575 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:24,614 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-14T09:44:24,614 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:24,614 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,614 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:24,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:44:24,682 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:24,719 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:24,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111403938def6520490bb2f84add3fb2d21f_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403938def6520490bb2f84add3fb2d21f_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:24,724 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/960cf8428fa546a9ae8fd8522bf45992, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:24,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/960cf8428fa546a9ae8fd8522bf45992 is 175, key is test_row_0/A:col10/1731577462128/Put/seqid=0 2024-11-14T09:44:24,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742040_1216 (size=31255) 2024-11-14T09:44:25,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/960cf8428fa546a9ae8fd8522bf45992 2024-11-14T09:44:25,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/1d15480e241244e28add945ebe77deb5 is 50, key is test_row_0/B:col10/1731577462128/Put/seqid=0 2024-11-14T09:44:25,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742041_1217 (size=12301) 2024-11-14T09:44:25,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/1d15480e241244e28add945ebe77deb5 2024-11-14T09:44:25,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/84611994cd924848905ae0f15776a0d7 is 50, key is test_row_0/C:col10/1731577462128/Put/seqid=0 2024-11-14T09:44:25,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742042_1218 (size=12301) 2024-11-14T09:44:25,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/84611994cd924848905ae0f15776a0d7 2024-11-14T09:44:25,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/960cf8428fa546a9ae8fd8522bf45992 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/960cf8428fa546a9ae8fd8522bf45992 2024-11-14T09:44:25,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/960cf8428fa546a9ae8fd8522bf45992, entries=150, sequenceid=383, filesize=30.5 K 2024-11-14T09:44:25,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/1d15480e241244e28add945ebe77deb5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/1d15480e241244e28add945ebe77deb5 2024-11-14T09:44:25,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/1d15480e241244e28add945ebe77deb5, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T09:44:25,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/84611994cd924848905ae0f15776a0d7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84611994cd924848905ae0f15776a0d7 2024-11-14T09:44:25,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84611994cd924848905ae0f15776a0d7, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T09:44:25,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 16957b2537beefdba492fe73e43233c2 in 1684ms, sequenceid=383, compaction requested=true 2024-11-14T09:44:25,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:25,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:25,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:25,987 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:25,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:25,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:25,987 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:25,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 16957b2537beefdba492fe73e43233c2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:25,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:25,988 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:25,988 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:25,988 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/A is initiating minor compaction (all files) 2024-11-14T09:44:25,988 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/B is initiating minor compaction (all files) 2024-11-14T09:44:25,988 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/A in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:25,988 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/B in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:25,988 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/b141daab0d9a4c4f9f0128bc859658f3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/ba560049145b4636abd708e1b84a3834, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/960cf8428fa546a9ae8fd8522bf45992] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=92.2 K 2024-11-14T09:44:25,989 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:25,989 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8c67d6f8887b493ab9ec9a0d209e9768, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/5a04eb65a423451688d303a96b658a7d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/1d15480e241244e28add945ebe77deb5] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.7 K 2024-11-14T09:44:25,989 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/b141daab0d9a4c4f9f0128bc859658f3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/ba560049145b4636abd708e1b84a3834, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/960cf8428fa546a9ae8fd8522bf45992] 2024-11-14T09:44:25,989 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b141daab0d9a4c4f9f0128bc859658f3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731577457741 2024-11-14T09:44:25,989 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c67d6f8887b493ab9ec9a0d209e9768, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731577457741 2024-11-14T09:44:25,989 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba560049145b4636abd708e1b84a3834, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731577459961 2024-11-14T09:44:25,989 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a04eb65a423451688d303a96b658a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731577459961 2024-11-14T09:44:25,989 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 960cf8428fa546a9ae8fd8522bf45992, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731577462128 2024-11-14T09:44:25,990 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d15480e241244e28add945ebe77deb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731577462128 2024-11-14T09:44:26,001 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:26,001 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#B#compaction#180 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:26,002 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/91ab0f215d6d4e3fb1f2f2c0b6bfe903 is 50, key is test_row_0/B:col10/1731577462128/Put/seqid=0 2024-11-14T09:44:26,004 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e602bb40d7aa4779bdb47d176837170e_16957b2537beefdba492fe73e43233c2 store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:26,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742043_1219 (size=13085) 2024-11-14T09:44:26,027 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e602bb40d7aa4779bdb47d176837170e_16957b2537beefdba492fe73e43233c2, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:26,027 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e602bb40d7aa4779bdb47d176837170e_16957b2537beefdba492fe73e43233c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:26,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742044_1220 (size=4469) 2024-11-14T09:44:26,411 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/91ab0f215d6d4e3fb1f2f2c0b6bfe903 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/91ab0f215d6d4e3fb1f2f2c0b6bfe903 2024-11-14T09:44:26,416 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/B of 16957b2537beefdba492fe73e43233c2 into 91ab0f215d6d4e3fb1f2f2c0b6bfe903(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:26,416 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:26,417 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/B, priority=13, startTime=1731577465987; duration=0sec 2024-11-14T09:44:26,417 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:26,417 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:B 2024-11-14T09:44:26,417 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:26,418 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:26,418 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 16957b2537beefdba492fe73e43233c2/C is initiating minor compaction (all files) 2024-11-14T09:44:26,418 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 16957b2537beefdba492fe73e43233c2/C in TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:26,418 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b69aedd2c5144fa3a540d415cc66c154, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/393ce7506dea4c1b98a3044804e4b424, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84611994cd924848905ae0f15776a0d7] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp, totalSize=36.7 K 2024-11-14T09:44:26,418 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b69aedd2c5144fa3a540d415cc66c154, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731577457741 2024-11-14T09:44:26,419 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 393ce7506dea4c1b98a3044804e4b424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731577459961 2024-11-14T09:44:26,419 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 84611994cd924848905ae0f15776a0d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731577462128 2024-11-14T09:44:26,428 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#C#compaction#182 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:26,428 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/69d913dbcad14539a514491864045b65 is 50, key is test_row_0/C:col10/1731577462128/Put/seqid=0 2024-11-14T09:44:26,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742045_1221 (size=13085) 2024-11-14T09:44:26,432 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 16957b2537beefdba492fe73e43233c2#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:26,433 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/24c7c52291f1463ca7f2a06cdd03da4a is 175, key is test_row_0/A:col10/1731577462128/Put/seqid=0 2024-11-14T09:44:26,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742046_1222 (size=32039) 2024-11-14T09:44:26,642 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:44:26,844 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/69d913dbcad14539a514491864045b65 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/69d913dbcad14539a514491864045b65 2024-11-14T09:44:26,845 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/24c7c52291f1463ca7f2a06cdd03da4a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/24c7c52291f1463ca7f2a06cdd03da4a 2024-11-14T09:44:26,850 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/C of 16957b2537beefdba492fe73e43233c2 into 69d913dbcad14539a514491864045b65(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:26,850 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 16957b2537beefdba492fe73e43233c2/A of 16957b2537beefdba492fe73e43233c2 into 24c7c52291f1463ca7f2a06cdd03da4a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:26,850 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:26,850 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 16957b2537beefdba492fe73e43233c2: 2024-11-14T09:44:26,850 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/C, priority=13, startTime=1731577465987; duration=0sec 2024-11-14T09:44:26,850 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., storeName=16957b2537beefdba492fe73e43233c2/A, priority=13, startTime=1731577465987; duration=0sec 2024-11-14T09:44:26,850 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:26,850 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:C 2024-11-14T09:44:26,850 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:26,850 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 16957b2537beefdba492fe73e43233c2:A 2024-11-14T09:44:28,442 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-14T09:44:28,442 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:28,442 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:28,442 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:28,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3259 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3056 2024-11-14T09:44:28,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:44:28,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1318 2024-11-14T09:44:28,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3951 rows 2024-11-14T09:44:28,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1318 2024-11-14T09:44:28,443 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3950 rows 2024-11-14T09:44:28,443 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:44:28,443 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:44:28,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:28,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:28,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:28,444 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:44:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:44:28,447 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577468447"}]},"ts":"1731577468447"} 2024-11-14T09:44:28,449 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:44:28,449 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:44:28,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:44:28,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, UNASSIGN}] 2024-11-14T09:44:28,452 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, UNASSIGN 2024-11-14T09:44:28,453 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:28,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, UNASSIGN because future has completed 2024-11-14T09:44:28,455 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:44:28,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:44:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:44:28,607 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:28,607 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:44:28,608 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 16957b2537beefdba492fe73e43233c2, disabling compactions & flushes 2024-11-14T09:44:28,608 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:28,608 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:28,608 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. after waiting 0 ms 2024-11-14T09:44:28,608 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:28,608 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 16957b2537beefdba492fe73e43233c2 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T09:44:28,608 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=A 2024-11-14T09:44:28,608 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:28,609 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=B 2024-11-14T09:44:28,609 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:28,609 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 16957b2537beefdba492fe73e43233c2, store=C 2024-11-14T09:44:28,609 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:28,619 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b30b9db8c8ab4dab839da6db5aa1bce5_16957b2537beefdba492fe73e43233c2 is 50, key is test_row_0/A:col10/1731577464611/Put/seqid=0 2024-11-14T09:44:28,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742047_1223 (size=12454) 2024-11-14T09:44:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:44:29,024 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:29,031 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b30b9db8c8ab4dab839da6db5aa1bce5_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b30b9db8c8ab4dab839da6db5aa1bce5_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:29,032 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/29f920dc8e294e9c9cba170b0d191e23, store: [table=TestAcidGuarantees family=A region=16957b2537beefdba492fe73e43233c2] 2024-11-14T09:44:29,033 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/29f920dc8e294e9c9cba170b0d191e23 is 175, key is test_row_0/A:col10/1731577464611/Put/seqid=0 2024-11-14T09:44:29,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742048_1224 (size=31255) 2024-11-14T09:44:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:44:29,438 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/29f920dc8e294e9c9cba170b0d191e23 2024-11-14T09:44:29,448 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/db941d2e4dc343b4b8848e46a8364161 is 50, key is test_row_0/B:col10/1731577464611/Put/seqid=0 2024-11-14T09:44:29,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742049_1225 (size=12301) 2024-11-14T09:44:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:44:29,852 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/db941d2e4dc343b4b8848e46a8364161 2024-11-14T09:44:29,864 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/e1e73ae4c68e415d98760d3fa8985eb0 is 50, key is test_row_0/C:col10/1731577464611/Put/seqid=0 2024-11-14T09:44:29,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742050_1226 (size=12301) 2024-11-14T09:44:30,270 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/e1e73ae4c68e415d98760d3fa8985eb0 2024-11-14T09:44:30,282 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/A/29f920dc8e294e9c9cba170b0d191e23 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/29f920dc8e294e9c9cba170b0d191e23 2024-11-14T09:44:30,287 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/29f920dc8e294e9c9cba170b0d191e23, entries=150, sequenceid=393, filesize=30.5 K 2024-11-14T09:44:30,288 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/B/db941d2e4dc343b4b8848e46a8364161 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/db941d2e4dc343b4b8848e46a8364161 2024-11-14T09:44:30,292 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/db941d2e4dc343b4b8848e46a8364161, entries=150, sequenceid=393, filesize=12.0 K 2024-11-14T09:44:30,293 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/.tmp/C/e1e73ae4c68e415d98760d3fa8985eb0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e1e73ae4c68e415d98760d3fa8985eb0 2024-11-14T09:44:30,296 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e1e73ae4c68e415d98760d3fa8985eb0, entries=150, sequenceid=393, filesize=12.0 K 2024-11-14T09:44:30,297 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 16957b2537beefdba492fe73e43233c2 in 1689ms, sequenceid=393, compaction requested=false 2024-11-14T09:44:30,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/58518b4d05334d61a2b76b2bb0bb740a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3dde710441074c8fbaf843969aaad4db, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/8bb9e1ce132c4fc69289b358d06fcc81, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/a069700ee8cb48699dd6d86166d86bf1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/492343f4078042dfb2d60038ecde328e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/13440659e117492d95b447dfb3e25f65, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc8bac464b4f4ba98d7a078443a45763, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/4f0118c79965498a90472c191efade24, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e32cccc055a14154bfea1ec8b1f71d35, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/d243711163fb468f8b4835e1b8e211f7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/23668b01432a4894948f82ef6dd974a8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/62b8a6d196964c2f848a437d1e988bab, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/82976a1a3eb74c20838c29a911a3af37, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc976e0fbf9447ef9d082abdeedab478, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e430bead306a4f3b8c11e4ce4ba40a57, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/1f07f14976e946c485ccb17a82336397, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/cb1b9f1728404ac4bfc4923a4a0aa2b1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/2d8bd5a12ba641029455571cdb0efb03, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/5d38a93adc96499185e2314ec9339b8a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3d1a0c97bb3c41dcbfdde0899355b354, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/b141daab0d9a4c4f9f0128bc859658f3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/ba560049145b4636abd708e1b84a3834, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/960cf8428fa546a9ae8fd8522bf45992] to archive 2024-11-14T09:44:30,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:44:30,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/58518b4d05334d61a2b76b2bb0bb740a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/58518b4d05334d61a2b76b2bb0bb740a 2024-11-14T09:44:30,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3dde710441074c8fbaf843969aaad4db to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3dde710441074c8fbaf843969aaad4db 2024-11-14T09:44:30,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/8bb9e1ce132c4fc69289b358d06fcc81 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/8bb9e1ce132c4fc69289b358d06fcc81 2024-11-14T09:44:30,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/a069700ee8cb48699dd6d86166d86bf1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/a069700ee8cb48699dd6d86166d86bf1 2024-11-14T09:44:30,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/492343f4078042dfb2d60038ecde328e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/492343f4078042dfb2d60038ecde328e 2024-11-14T09:44:30,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/13440659e117492d95b447dfb3e25f65 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/13440659e117492d95b447dfb3e25f65 2024-11-14T09:44:30,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc8bac464b4f4ba98d7a078443a45763 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc8bac464b4f4ba98d7a078443a45763 2024-11-14T09:44:30,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/4f0118c79965498a90472c191efade24 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/4f0118c79965498a90472c191efade24 2024-11-14T09:44:30,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e32cccc055a14154bfea1ec8b1f71d35 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e32cccc055a14154bfea1ec8b1f71d35 2024-11-14T09:44:30,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/d243711163fb468f8b4835e1b8e211f7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/d243711163fb468f8b4835e1b8e211f7 2024-11-14T09:44:30,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/23668b01432a4894948f82ef6dd974a8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/23668b01432a4894948f82ef6dd974a8 2024-11-14T09:44:30,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/62b8a6d196964c2f848a437d1e988bab to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/62b8a6d196964c2f848a437d1e988bab 2024-11-14T09:44:30,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/82976a1a3eb74c20838c29a911a3af37 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/82976a1a3eb74c20838c29a911a3af37 2024-11-14T09:44:30,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc976e0fbf9447ef9d082abdeedab478 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/dc976e0fbf9447ef9d082abdeedab478 2024-11-14T09:44:30,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e430bead306a4f3b8c11e4ce4ba40a57 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/e430bead306a4f3b8c11e4ce4ba40a57 2024-11-14T09:44:30,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/1f07f14976e946c485ccb17a82336397 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/1f07f14976e946c485ccb17a82336397 2024-11-14T09:44:30,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/cb1b9f1728404ac4bfc4923a4a0aa2b1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/cb1b9f1728404ac4bfc4923a4a0aa2b1 2024-11-14T09:44:30,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/2d8bd5a12ba641029455571cdb0efb03 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/2d8bd5a12ba641029455571cdb0efb03 2024-11-14T09:44:30,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/5d38a93adc96499185e2314ec9339b8a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/5d38a93adc96499185e2314ec9339b8a 2024-11-14T09:44:30,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3d1a0c97bb3c41dcbfdde0899355b354 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/3d1a0c97bb3c41dcbfdde0899355b354 2024-11-14T09:44:30,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/b141daab0d9a4c4f9f0128bc859658f3 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/b141daab0d9a4c4f9f0128bc859658f3 2024-11-14T09:44:30,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/ba560049145b4636abd708e1b84a3834 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/ba560049145b4636abd708e1b84a3834 2024-11-14T09:44:30,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/960cf8428fa546a9ae8fd8522bf45992 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/960cf8428fa546a9ae8fd8522bf45992 2024-11-14T09:44:30,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/88a1b3abb0104edf9ecebf8bc5fd5e35, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/3169810c41dc49408bb22af78443c130, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0d9d31e620c3411dab12f98458942f6f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/658dca2c8a4b48f7b6418d3f5b58d83f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8ecdd9c143504ae29d627a4555bc667c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/cdd5e7811eca4f75af75c6b30c64fca7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/c01a44db5c2545978b3ab055a8587858, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f1afcd96921141c087bfd0a9996399d1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/9bb59f6bc3684c50bbac9bc70bbbc4b1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2124c51e467546ed9f3466d826aad1ee, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/df11dfdf0c7e4f23bcd7d4f532c66b97, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2370024161854191935e01016fc01952, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/b4bb7216bb5e4091b0d75808c4bf19bd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/645276ba3e8848fdb789607a2e15ba18, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/6558076f69074356b7ff11dfaf28eacf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0c9985543aa140fbbc43ed63644ce04c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d0ac4342b3f74b25bd070eae3315e2b4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f240faa7cb2748e480d0519009eee415, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0b8b5bb759064c07b16389bb12ca393e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8c67d6f8887b493ab9ec9a0d209e9768, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d1f39508e4ae455a8e15ddbec5695837, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/5a04eb65a423451688d303a96b658a7d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/1d15480e241244e28add945ebe77deb5] to archive 2024-11-14T09:44:30,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:44:30,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/88a1b3abb0104edf9ecebf8bc5fd5e35 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/88a1b3abb0104edf9ecebf8bc5fd5e35 2024-11-14T09:44:30,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/3169810c41dc49408bb22af78443c130 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/3169810c41dc49408bb22af78443c130 2024-11-14T09:44:30,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0d9d31e620c3411dab12f98458942f6f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0d9d31e620c3411dab12f98458942f6f 2024-11-14T09:44:30,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/658dca2c8a4b48f7b6418d3f5b58d83f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/658dca2c8a4b48f7b6418d3f5b58d83f 2024-11-14T09:44:30,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8ecdd9c143504ae29d627a4555bc667c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8ecdd9c143504ae29d627a4555bc667c 2024-11-14T09:44:30,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/cdd5e7811eca4f75af75c6b30c64fca7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/cdd5e7811eca4f75af75c6b30c64fca7 2024-11-14T09:44:30,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/c01a44db5c2545978b3ab055a8587858 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/c01a44db5c2545978b3ab055a8587858 2024-11-14T09:44:30,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f1afcd96921141c087bfd0a9996399d1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f1afcd96921141c087bfd0a9996399d1 2024-11-14T09:44:30,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/9bb59f6bc3684c50bbac9bc70bbbc4b1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/9bb59f6bc3684c50bbac9bc70bbbc4b1 2024-11-14T09:44:30,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2124c51e467546ed9f3466d826aad1ee to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2124c51e467546ed9f3466d826aad1ee 2024-11-14T09:44:30,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/df11dfdf0c7e4f23bcd7d4f532c66b97 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/df11dfdf0c7e4f23bcd7d4f532c66b97 2024-11-14T09:44:30,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2370024161854191935e01016fc01952 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/2370024161854191935e01016fc01952 2024-11-14T09:44:30,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/b4bb7216bb5e4091b0d75808c4bf19bd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/b4bb7216bb5e4091b0d75808c4bf19bd 2024-11-14T09:44:30,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/645276ba3e8848fdb789607a2e15ba18 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/645276ba3e8848fdb789607a2e15ba18 2024-11-14T09:44:30,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/6558076f69074356b7ff11dfaf28eacf to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/6558076f69074356b7ff11dfaf28eacf 2024-11-14T09:44:30,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0c9985543aa140fbbc43ed63644ce04c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0c9985543aa140fbbc43ed63644ce04c 2024-11-14T09:44:30,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d0ac4342b3f74b25bd070eae3315e2b4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d0ac4342b3f74b25bd070eae3315e2b4 2024-11-14T09:44:30,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f240faa7cb2748e480d0519009eee415 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/f240faa7cb2748e480d0519009eee415 2024-11-14T09:44:30,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0b8b5bb759064c07b16389bb12ca393e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/0b8b5bb759064c07b16389bb12ca393e 2024-11-14T09:44:30,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8c67d6f8887b493ab9ec9a0d209e9768 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/8c67d6f8887b493ab9ec9a0d209e9768 2024-11-14T09:44:30,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d1f39508e4ae455a8e15ddbec5695837 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/d1f39508e4ae455a8e15ddbec5695837 2024-11-14T09:44:30,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/5a04eb65a423451688d303a96b658a7d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/5a04eb65a423451688d303a96b658a7d 2024-11-14T09:44:30,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/1d15480e241244e28add945ebe77deb5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/1d15480e241244e28add945ebe77deb5 2024-11-14T09:44:30,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/58f399f4c97f4e6284e50403022187d1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/02aa8bb257464a7183a8e74272171f43, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b8f105852fec404b92f8726b1a64b75f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/112d9c25340b4686910b8b182fe5b263, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/76368326655c42febc20d9d312e5432a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/a108dfdea5754045905de4d42077a5fd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/0de21772cd5c4dc483ad4ead30738a45, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84ae62dbcb3144a6b537ec18be26abab, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c906ae0410214dd58fce09aa922dc58e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e7d071f46cc249d4ad7f0e74bb0222af, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/19364593e6ed45c7bf8aec02a84bf123, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/806ccc6e90c64cc895cc3af6e8682b7f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/5c2b76545505462f990f82d6ebe81890, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/188be3502d0f4dfe8b1d448df94d1822, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c94254dcb579409093d764e27998d14b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4b2e2536aeb94670b87098c8a3fa481b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c607d986469a42ca8a865fa89ccae990, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/7c687b2972234c2fb24ec7852b2572de, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/61e9e0be696d4c2b84e9a4a75c34b42e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b69aedd2c5144fa3a540d415cc66c154, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4741bc71e7894b51afc5bb97f14dff5b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/393ce7506dea4c1b98a3044804e4b424, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84611994cd924848905ae0f15776a0d7] to archive 2024-11-14T09:44:30,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:44:30,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/58f399f4c97f4e6284e50403022187d1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/58f399f4c97f4e6284e50403022187d1 2024-11-14T09:44:30,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/02aa8bb257464a7183a8e74272171f43 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/02aa8bb257464a7183a8e74272171f43 2024-11-14T09:44:30,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b8f105852fec404b92f8726b1a64b75f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b8f105852fec404b92f8726b1a64b75f 2024-11-14T09:44:30,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/112d9c25340b4686910b8b182fe5b263 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/112d9c25340b4686910b8b182fe5b263 2024-11-14T09:44:30,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/76368326655c42febc20d9d312e5432a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/76368326655c42febc20d9d312e5432a 2024-11-14T09:44:30,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/a108dfdea5754045905de4d42077a5fd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/a108dfdea5754045905de4d42077a5fd 2024-11-14T09:44:30,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/0de21772cd5c4dc483ad4ead30738a45 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/0de21772cd5c4dc483ad4ead30738a45 2024-11-14T09:44:30,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84ae62dbcb3144a6b537ec18be26abab to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84ae62dbcb3144a6b537ec18be26abab 2024-11-14T09:44:30,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c906ae0410214dd58fce09aa922dc58e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c906ae0410214dd58fce09aa922dc58e 2024-11-14T09:44:30,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e7d071f46cc249d4ad7f0e74bb0222af to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e7d071f46cc249d4ad7f0e74bb0222af 2024-11-14T09:44:30,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/19364593e6ed45c7bf8aec02a84bf123 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/19364593e6ed45c7bf8aec02a84bf123 2024-11-14T09:44:30,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/806ccc6e90c64cc895cc3af6e8682b7f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/806ccc6e90c64cc895cc3af6e8682b7f 2024-11-14T09:44:30,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/5c2b76545505462f990f82d6ebe81890 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/5c2b76545505462f990f82d6ebe81890 2024-11-14T09:44:30,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/188be3502d0f4dfe8b1d448df94d1822 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/188be3502d0f4dfe8b1d448df94d1822 2024-11-14T09:44:30,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c94254dcb579409093d764e27998d14b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c94254dcb579409093d764e27998d14b 2024-11-14T09:44:30,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4b2e2536aeb94670b87098c8a3fa481b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4b2e2536aeb94670b87098c8a3fa481b 2024-11-14T09:44:30,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c607d986469a42ca8a865fa89ccae990 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/c607d986469a42ca8a865fa89ccae990 2024-11-14T09:44:30,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/7c687b2972234c2fb24ec7852b2572de to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/7c687b2972234c2fb24ec7852b2572de 2024-11-14T09:44:30,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/61e9e0be696d4c2b84e9a4a75c34b42e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/61e9e0be696d4c2b84e9a4a75c34b42e 2024-11-14T09:44:30,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b69aedd2c5144fa3a540d415cc66c154 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/b69aedd2c5144fa3a540d415cc66c154 2024-11-14T09:44:30,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4741bc71e7894b51afc5bb97f14dff5b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/4741bc71e7894b51afc5bb97f14dff5b 2024-11-14T09:44:30,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/393ce7506dea4c1b98a3044804e4b424 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/393ce7506dea4c1b98a3044804e4b424 2024-11-14T09:44:30,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84611994cd924848905ae0f15776a0d7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/84611994cd924848905ae0f15776a0d7 2024-11-14T09:44:30,370 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=4 2024-11-14T09:44:30,371 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. 2024-11-14T09:44:30,371 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 16957b2537beefdba492fe73e43233c2: Waiting for close lock at 1731577468607Running coprocessor pre-close hooks at 1731577468607Disabling compacts and flushes for region at 1731577468607Disabling writes for close at 1731577468608 (+1 ms)Obtaining lock to block concurrent updates at 1731577468608Preparing flush snapshotting stores in 16957b2537beefdba492fe73e43233c2 at 1731577468608Finished memstore snapshotting TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731577468609 (+1 ms)Flushing stores of TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2. at 1731577468610 (+1 ms)Flushing 16957b2537beefdba492fe73e43233c2/A: creating writer at 1731577468610Flushing mob file 16957b2537beefdba492fe73e43233c2/A: appending metadata at 1731577468619 (+9 ms)Flushing mob file 16957b2537beefdba492fe73e43233c2/A: closing flushed file at 1731577468619Flushing 16957b2537beefdba492fe73e43233c2/A: appending metadata at 1731577469033 (+414 ms)Flushing 16957b2537beefdba492fe73e43233c2/A: closing flushed file at 1731577469033Flushing 16957b2537beefdba492fe73e43233c2/B: creating writer at 1731577469445 (+412 ms)Flushing 16957b2537beefdba492fe73e43233c2/B: appending metadata at 1731577469447 (+2 ms)Flushing 16957b2537beefdba492fe73e43233c2/B: closing flushed file at 1731577469448 (+1 ms)Flushing 16957b2537beefdba492fe73e43233c2/C: creating writer at 1731577469860 (+412 ms)Flushing 16957b2537beefdba492fe73e43233c2/C: appending metadata at 1731577469863 (+3 ms)Flushing 16957b2537beefdba492fe73e43233c2/C: closing flushed file at 1731577469863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54ff6762: reopening flushed file at 1731577470281 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53c0c8a0: reopening flushed file at 1731577470287 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e69686f: reopening flushed file at 1731577470292 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 16957b2537beefdba492fe73e43233c2 in 1689ms, sequenceid=393, compaction requested=false at 1731577470297 (+5 ms)Writing region close event to WAL at 1731577470367 (+70 ms)Running coprocessor post-close hooks at 1731577470371 (+4 ms)Closed at 1731577470371 2024-11-14T09:44:30,373 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,373 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=16957b2537beefdba492fe73e43233c2, regionState=CLOSED 2024-11-14T09:44:30,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:44:30,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T09:44:30,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 16957b2537beefdba492fe73e43233c2, server=defc576eb6b7,42511,1731577408298 in 1.9200 sec 2024-11-14T09:44:30,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-14T09:44:30,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=16957b2537beefdba492fe73e43233c2, UNASSIGN in 1.9250 sec 2024-11-14T09:44:30,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T09:44:30,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9280 sec 2024-11-14T09:44:30,380 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577470380"}]},"ts":"1731577470380"} 2024-11-14T09:44:30,381 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:44:30,381 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:44:30,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-14T09:44:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:44:30,591 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:30,592 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:44:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:30,595 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:44:30,596 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:30,600 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,604 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/recovered.edits] 2024-11-14T09:44:30,607 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/24c7c52291f1463ca7f2a06cdd03da4a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/24c7c52291f1463ca7f2a06cdd03da4a 2024-11-14T09:44:30,609 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/29f920dc8e294e9c9cba170b0d191e23 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/A/29f920dc8e294e9c9cba170b0d191e23 2024-11-14T09:44:30,611 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/91ab0f215d6d4e3fb1f2f2c0b6bfe903 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/91ab0f215d6d4e3fb1f2f2c0b6bfe903 2024-11-14T09:44:30,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/db941d2e4dc343b4b8848e46a8364161 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/B/db941d2e4dc343b4b8848e46a8364161 2024-11-14T09:44:30,616 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/69d913dbcad14539a514491864045b65 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/69d913dbcad14539a514491864045b65 2024-11-14T09:44:30,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e1e73ae4c68e415d98760d3fa8985eb0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/C/e1e73ae4c68e415d98760d3fa8985eb0 2024-11-14T09:44:30,619 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/recovered.edits/396.seqid to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2/recovered.edits/396.seqid 2024-11-14T09:44:30,620 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,620 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:44:30,620 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:44:30,621 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:44:30,624 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403938def6520490bb2f84add3fb2d21f_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403938def6520490bb2f84add3fb2d21f_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,625 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111415cee01f25984be081a0309285e9c51a_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111415cee01f25984be081a0309285e9c51a_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,626 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114291cc8bc3b544aabb51e6f264aa2138f_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114291cc8bc3b544aabb51e6f264aa2138f_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,627 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111440970b7ca95d41898f6049045c7454d9_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111440970b7ca95d41898f6049045c7454d9_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,628 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114485bd38797d34626a0c1c907add32652_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114485bd38797d34626a0c1c907add32652_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,630 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145295a16566d04c81a5483691dc14dd3d_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145295a16566d04c81a5483691dc14dd3d_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,631 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146ac5333cc2994dd396ef1d40aab0849f_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146ac5333cc2994dd396ef1d40aab0849f_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,632 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111474ee3aacc95f4b6da404e909f00e8321_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111474ee3aacc95f4b6da404e909f00e8321_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,633 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b6096d126f3459fb19f43f087eb1b2a_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b6096d126f3459fb19f43f087eb1b2a_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,635 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3ef05e1073d41718a09b5a8c6df768c_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3ef05e1073d41718a09b5a8c6df768c_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,636 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aaa3d57887914dbcb56448c1081a4cf1_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114aaa3d57887914dbcb56448c1081a4cf1_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,637 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114adf55cbbd36848daa9caf3d2a0fa15f9_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114adf55cbbd36848daa9caf3d2a0fa15f9_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,638 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1f3b9c945e248de9549710987e032fe_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1f3b9c945e248de9549710987e032fe_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,639 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b30b9db8c8ab4dab839da6db5aa1bce5_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b30b9db8c8ab4dab839da6db5aa1bce5_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,641 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b7133d421e234f49bb566e1b7e8815c4_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b7133d421e234f49bb566e1b7e8815c4_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,642 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bd05f70b89a64f5ca55a91dfe0798da4_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bd05f70b89a64f5ca55a91dfe0798da4_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,643 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e34b704ea74b6d8b3c99fedc0c600d_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e34b704ea74b6d8b3c99fedc0c600d_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,644 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114da08131be94140bf9c15fad702636900_16957b2537beefdba492fe73e43233c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114da08131be94140bf9c15fad702636900_16957b2537beefdba492fe73e43233c2 2024-11-14T09:44:30,644 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:44:30,647 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:30,650 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:44:30,653 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:44:30,654 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:30,654 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:44:30,655 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577470654"}]},"ts":"9223372036854775807"} 2024-11-14T09:44:30,658 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:44:30,658 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 16957b2537beefdba492fe73e43233c2, NAME => 'TestAcidGuarantees,,1731577439904.16957b2537beefdba492fe73e43233c2.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:44:30,658 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:44:30,658 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577470658"}]},"ts":"9223372036854775807"} 2024-11-14T09:44:30,660 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:44:30,661 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:30,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-14T09:44:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:44:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:44:30,701 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:30,715 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-969773813_22 at /127.0.0.1:55652 [Waiting for operation #154] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-969773813_22 at /127.0.0.1:55310 [Waiting for operation #262] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-572166304_22 at /127.0.0.1:55654 [Waiting for operation #156] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-572166304_22 at /127.0.0.1:59604 [Waiting for operation #271] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 416), ProcessCount=11 (was 11), AvailableMemoryMB=3905 (was 4080) 2024-11-14T09:44:30,725 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=369, ProcessCount=11, AvailableMemoryMB=3905 2024-11-14T09:44:30,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:44:30,726 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:44:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:44:30,728 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:44:30,729 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:30,729 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-14T09:44:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T09:44:30,730 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:44:30,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742051_1227 (size=1061) 2024-11-14T09:44:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T09:44:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T09:44:31,141 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:44:31,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742052_1228 (size=53) 2024-11-14T09:44:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T09:44:31,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:44:31,551 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ab93aec2a07aaf72eae598338b2d22a6, disabling compactions & flushes 2024-11-14T09:44:31,551 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. after waiting 0 ms 2024-11-14T09:44:31,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,552 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab93aec2a07aaf72eae598338b2d22a6: Waiting for close lock at 1731577471551Disabling compacts and flushes for region at 1731577471551Disabling writes for close at 1731577471552 (+1 ms)Writing region close event to WAL at 1731577471552Closed at 1731577471552 2024-11-14T09:44:31,553 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:44:31,553 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577471553"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577471553"}]},"ts":"1731577471553"} 2024-11-14T09:44:31,555 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:44:31,556 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:44:31,557 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577471556"}]},"ts":"1731577471556"} 2024-11-14T09:44:31,559 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:44:31,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, ASSIGN}] 2024-11-14T09:44:31,561 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, ASSIGN 2024-11-14T09:44:31,562 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, ASSIGN; state=OFFLINE, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=false 2024-11-14T09:44:31,713 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=ab93aec2a07aaf72eae598338b2d22a6, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:31,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, ASSIGN because future has completed 2024-11-14T09:44:31,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:44:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T09:44:31,879 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,879 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:44:31,879 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,879 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:44:31,879 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,880 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,881 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,883 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:31,883 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab93aec2a07aaf72eae598338b2d22a6 columnFamilyName A 2024-11-14T09:44:31,883 DEBUG [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:31,884 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.HStore(327): Store=ab93aec2a07aaf72eae598338b2d22a6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:31,884 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,886 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:31,886 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab93aec2a07aaf72eae598338b2d22a6 columnFamilyName B 2024-11-14T09:44:31,886 DEBUG [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:31,887 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.HStore(327): Store=ab93aec2a07aaf72eae598338b2d22a6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:31,887 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,888 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:44:31,889 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab93aec2a07aaf72eae598338b2d22a6 columnFamilyName C 2024-11-14T09:44:31,889 DEBUG [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:44:31,889 INFO [StoreOpener-ab93aec2a07aaf72eae598338b2d22a6-1 {}] regionserver.HStore(327): Store=ab93aec2a07aaf72eae598338b2d22a6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:44:31,890 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,890 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,890 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,891 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,892 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,892 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,893 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:44:31,894 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,896 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:44:31,897 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened ab93aec2a07aaf72eae598338b2d22a6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62304541, jitterRate=-0.07158999145030975}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:44:31,897 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:31,898 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for ab93aec2a07aaf72eae598338b2d22a6: Running coprocessor pre-open hook at 1731577471880Writing region info on filesystem at 1731577471880Initializing all the Stores at 1731577471881 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577471881Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577471881Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577471881Cleaning up temporary data from old regions at 1731577471892 (+11 ms)Running coprocessor post-open hooks at 1731577471897 (+5 ms)Region opened successfully at 1731577471898 (+1 ms) 2024-11-14T09:44:31,899 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., pid=55, masterSystemTime=1731577471871 2024-11-14T09:44:31,901 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,901 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:31,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=ab93aec2a07aaf72eae598338b2d22a6, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:31,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:44:31,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T09:44:31,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 in 188 msec 2024-11-14T09:44:31,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-14T09:44:31,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, ASSIGN in 349 msec 2024-11-14T09:44:31,911 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:44:31,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577471911"}]},"ts":"1731577471911"} 2024-11-14T09:44:31,913 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:44:31,914 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:44:31,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-14T09:44:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T09:44:32,881 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:32,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52272536, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37284d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,890 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,892 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:44:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,33347,1731577407591 2024-11-14T09:44:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7530f462 2024-11-14T09:44:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:44:32,894 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:44:32,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa1a6f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20872555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d979ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da84a18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228a5bd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a917929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,906 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c7f863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,907 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3356246a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b66d8b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,911 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9793eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,913 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4173c014, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,914 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59051518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f85b15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,918 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a888d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,920 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,921 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446786fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,924 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b95e807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5403e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:44:32,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:44:32,927 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:44:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:44:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:44:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee8448a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:44:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:44:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:32,928 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:44:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8be68a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:44:32,929 DEBUG [Thread-1087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,929 DEBUG [Thread-1076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,929 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,929 DEBUG [Thread-1085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,929 DEBUG [Thread-1078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,930 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,930 DEBUG [Thread-1076 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,930 DEBUG [Thread-1076 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,930 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,930 DEBUG [Thread-1087 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-14T09:44:32,931 DEBUG [Thread-1087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,932 DEBUG [Thread-1089 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,932 DEBUG [Thread-1089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,932 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:44:32,933 DEBUG [Thread-1070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,933 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:32,934 DEBUG [Thread-1072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,934 DEBUG [Thread-1074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:44:32,935 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:44:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:44:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:44:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:44:32,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:44:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:32,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/5a421938c92f416fa7b0fc9534b0b8c4 is 50, key is test_row_0/A:col10/1731577472957/Put/seqid=0 2024-11-14T09:44:32,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577532984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:32,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577532990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:32,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577532991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577532992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742053_1229 (size=12001) 2024-11-14T09:44:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:32,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577532993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:32,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/5a421938c92f416fa7b0fc9534b0b8c4 2024-11-14T09:44:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:32,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/bfe18b16ae204585bb55d4b48b6f32b7 is 50, key is test_row_0/B:col10/1731577472957/Put/seqid=0 2024-11-14T09:44:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742054_1230 (size=12001) 2024-11-14T09:44:33,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/bfe18b16ae204585bb55d4b48b6f32b7 2024-11-14T09:44:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:44:33,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/895bda11c8b14dbc856282d35350a27d is 50, key is test_row_0/C:col10/1731577472957/Put/seqid=0 2024-11-14T09:44:33,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T09:44:33,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:33,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:33,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:33,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:33,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:33,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577533101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577533101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577533101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577533101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577533110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742055_1231 (size=12001) 2024-11-14T09:44:33,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/895bda11c8b14dbc856282d35350a27d 2024-11-14T09:44:33,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/5a421938c92f416fa7b0fc9534b0b8c4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5a421938c92f416fa7b0fc9534b0b8c4 2024-11-14T09:44:33,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5a421938c92f416fa7b0fc9534b0b8c4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:44:33,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/bfe18b16ae204585bb55d4b48b6f32b7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bfe18b16ae204585bb55d4b48b6f32b7 2024-11-14T09:44:33,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bfe18b16ae204585bb55d4b48b6f32b7, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:44:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/895bda11c8b14dbc856282d35350a27d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/895bda11c8b14dbc856282d35350a27d 2024-11-14T09:44:33,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/895bda11c8b14dbc856282d35350a27d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:44:33,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ab93aec2a07aaf72eae598338b2d22a6 in 207ms, sequenceid=14, compaction requested=false 2024-11-14T09:44:33,165 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:44:33,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:33,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T09:44:33,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:33,245 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:44:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:33,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/0e4323b7aa4a466da6adc644cb920738 is 50, key is test_row_0/A:col10/1731577472991/Put/seqid=0 2024-11-14T09:44:33,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:44:33,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742056_1232 (size=12001) 2024-11-14T09:44:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577533348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577533349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577533350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577533350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577533353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577533460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577533460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577533460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577533460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577533460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:44:33,666 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/0e4323b7aa4a466da6adc644cb920738 2024-11-14T09:44:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577533670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577533671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577533671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/52b9841b5533455090e943d85e49c5c4 is 50, key is test_row_0/B:col10/1731577472991/Put/seqid=0 2024-11-14T09:44:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577533671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577533671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742057_1233 (size=12001) 2024-11-14T09:44:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577533980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577533980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577533980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577533980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577533980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:44:34,077 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/52b9841b5533455090e943d85e49c5c4 2024-11-14T09:44:34,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/2fce20641cb84105a5c00186c5aedac2 is 50, key is test_row_0/C:col10/1731577472991/Put/seqid=0 2024-11-14T09:44:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742058_1234 (size=12001) 2024-11-14T09:44:34,488 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/2fce20641cb84105a5c00186c5aedac2 2024-11-14T09:44:34,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577534490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577534491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577534491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577534491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577534491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/0e4323b7aa4a466da6adc644cb920738 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0e4323b7aa4a466da6adc644cb920738 2024-11-14T09:44:34,498 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0e4323b7aa4a466da6adc644cb920738, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:44:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/52b9841b5533455090e943d85e49c5c4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/52b9841b5533455090e943d85e49c5c4 2024-11-14T09:44:34,503 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/52b9841b5533455090e943d85e49c5c4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:44:34,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/2fce20641cb84105a5c00186c5aedac2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/2fce20641cb84105a5c00186c5aedac2 2024-11-14T09:44:34,509 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/2fce20641cb84105a5c00186c5aedac2, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:44:34,509 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ab93aec2a07aaf72eae598338b2d22a6 in 1264ms, sequenceid=33, compaction requested=false 2024-11-14T09:44:34,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:34,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:34,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-14T09:44:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-14T09:44:34,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T09:44:34,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-14T09:44:34,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.5830 sec 2024-11-14T09:44:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T09:44:35,080 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:35,081 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-14T09:44:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:44:35,083 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:35,084 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:35,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:44:35,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T09:44:35,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:35,236 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:44:35,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:35,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:35,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:35,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:35,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:35,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:35,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/4c4cde2d77df49a9a14f291e09046b43 is 50, key is test_row_0/A:col10/1731577473313/Put/seqid=0 2024-11-14T09:44:35,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742059_1235 (size=12001) 2024-11-14T09:44:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:44:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:35,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577535517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577535517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577535518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577535520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577535521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577535630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577535631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577535631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577535631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577535631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,645 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/4c4cde2d77df49a9a14f291e09046b43 2024-11-14T09:44:35,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/010d031a4bf049eaba7172f44d0f3909 is 50, key is test_row_0/B:col10/1731577473313/Put/seqid=0 2024-11-14T09:44:35,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742060_1236 (size=12001) 2024-11-14T09:44:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:44:35,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577535840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577535840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577535840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577535840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577535841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:35,905 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:44:36,057 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/010d031a4bf049eaba7172f44d0f3909 2024-11-14T09:44:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4ffd520b24154da182e74921a9f2fbef is 50, key is test_row_0/C:col10/1731577473313/Put/seqid=0 2024-11-14T09:44:36,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742061_1237 (size=12001) 2024-11-14T09:44:36,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577536150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577536150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577536150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577536150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577536150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:44:36,473 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4ffd520b24154da182e74921a9f2fbef 2024-11-14T09:44:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/4c4cde2d77df49a9a14f291e09046b43 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/4c4cde2d77df49a9a14f291e09046b43 2024-11-14T09:44:36,482 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/4c4cde2d77df49a9a14f291e09046b43, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T09:44:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/010d031a4bf049eaba7172f44d0f3909 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/010d031a4bf049eaba7172f44d0f3909 2024-11-14T09:44:36,486 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/010d031a4bf049eaba7172f44d0f3909, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T09:44:36,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4ffd520b24154da182e74921a9f2fbef as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4ffd520b24154da182e74921a9f2fbef 2024-11-14T09:44:36,491 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4ffd520b24154da182e74921a9f2fbef, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T09:44:36,492 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ab93aec2a07aaf72eae598338b2d22a6 in 1256ms, sequenceid=46, compaction requested=true 2024-11-14T09:44:36,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:36,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:36,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-14T09:44:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-14T09:44:36,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T09:44:36,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-14T09:44:36,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4150 sec 2024-11-14T09:44:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:36,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:44:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:36,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/edd105e13aa440e3808816672349b892 is 50, key is test_row_0/A:col10/1731577475516/Put/seqid=0 2024-11-14T09:44:36,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742062_1238 (size=14341) 2024-11-14T09:44:36,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577536672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577536672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577536673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577536674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577536675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577536780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577536780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577536790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577536790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577536790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577536990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577536990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577537000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577537000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577537000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/edd105e13aa440e3808816672349b892 2024-11-14T09:44:37,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/7cd6b7b7a14b4f19b6d5845ce65d2623 is 50, key is test_row_0/B:col10/1731577475516/Put/seqid=0 2024-11-14T09:44:37,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742063_1239 (size=12001) 2024-11-14T09:44:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:44:37,230 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:37,231 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-14T09:44:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:44:37,232 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:37,233 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:37,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:37,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577537301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577537301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577537310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577537310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577537311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:44:37,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T09:44:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/7cd6b7b7a14b4f19b6d5845ce65d2623 2024-11-14T09:44:37,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/25ad8e9a128c4d3b822fd212c4d35fa8 is 50, key is test_row_0/C:col10/1731577475516/Put/seqid=0 2024-11-14T09:44:37,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742064_1240 (size=12001) 2024-11-14T09:44:37,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T09:44:37,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:37,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:44:37,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T09:44:37,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577537810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577537820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577537820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577537820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577537821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:37,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T09:44:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:44:37,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/25ad8e9a128c4d3b822fd212c4d35fa8 2024-11-14T09:44:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/edd105e13aa440e3808816672349b892 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/edd105e13aa440e3808816672349b892 2024-11-14T09:44:37,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/edd105e13aa440e3808816672349b892, entries=200, sequenceid=66, filesize=14.0 K 2024-11-14T09:44:37,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/7cd6b7b7a14b4f19b6d5845ce65d2623 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/7cd6b7b7a14b4f19b6d5845ce65d2623 2024-11-14T09:44:37,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/7cd6b7b7a14b4f19b6d5845ce65d2623, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T09:44:37,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/25ad8e9a128c4d3b822fd212c4d35fa8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/25ad8e9a128c4d3b822fd212c4d35fa8 2024-11-14T09:44:37,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/25ad8e9a128c4d3b822fd212c4d35fa8, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T09:44:37,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ab93aec2a07aaf72eae598338b2d22a6 in 1257ms, sequenceid=66, compaction requested=true 2024-11-14T09:44:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:37,921 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:37,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:37,921 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:37,923 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:37,923 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/A is initiating minor compaction (all files) 2024-11-14T09:44:37,923 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/A in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,923 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5a421938c92f416fa7b0fc9534b0b8c4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0e4323b7aa4a466da6adc644cb920738, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/4c4cde2d77df49a9a14f291e09046b43, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/edd105e13aa440e3808816672349b892] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=49.2 K 2024-11-14T09:44:37,923 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:37,923 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/B is initiating minor compaction (all files) 2024-11-14T09:44:37,923 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/B in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,923 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bfe18b16ae204585bb55d4b48b6f32b7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/52b9841b5533455090e943d85e49c5c4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/010d031a4bf049eaba7172f44d0f3909, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/7cd6b7b7a14b4f19b6d5845ce65d2623] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=46.9 K 2024-11-14T09:44:37,924 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a421938c92f416fa7b0fc9534b0b8c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577472947 2024-11-14T09:44:37,924 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe18b16ae204585bb55d4b48b6f32b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577472947 2024-11-14T09:44:37,924 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4323b7aa4a466da6adc644cb920738, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731577472977 2024-11-14T09:44:37,924 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 52b9841b5533455090e943d85e49c5c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731577472977 2024-11-14T09:44:37,924 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c4cde2d77df49a9a14f291e09046b43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577473313 2024-11-14T09:44:37,925 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 010d031a4bf049eaba7172f44d0f3909, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577473313 2024-11-14T09:44:37,925 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting edd105e13aa440e3808816672349b892, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577475516 2024-11-14T09:44:37,925 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd6b7b7a14b4f19b6d5845ce65d2623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577475516 2024-11-14T09:44:37,935 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#A#compaction#198 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:37,936 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/45ef2effe8fc4c53be226a2ea22466f3 is 50, key is test_row_0/A:col10/1731577475516/Put/seqid=0 2024-11-14T09:44:37,939 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#B#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:37,940 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/1793b1e6aecb429fb92ab9fd9e3c36bf is 50, key is test_row_0/B:col10/1731577475516/Put/seqid=0 2024-11-14T09:44:37,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742065_1241 (size=12139) 2024-11-14T09:44:37,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742066_1242 (size=12139) 2024-11-14T09:44:37,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T09:44:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:37,996 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:44:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:38,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/31a34375a49a4b5e9411e38219b555ba is 50, key is test_row_0/A:col10/1731577476666/Put/seqid=0 2024-11-14T09:44:38,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742067_1243 (size=12001) 2024-11-14T09:44:38,351 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/45ef2effe8fc4c53be226a2ea22466f3 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/45ef2effe8fc4c53be226a2ea22466f3 2024-11-14T09:44:38,357 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/A of ab93aec2a07aaf72eae598338b2d22a6 into 45ef2effe8fc4c53be226a2ea22466f3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:38,358 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:38,358 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/A, priority=12, startTime=1731577477921; duration=0sec 2024-11-14T09:44:38,358 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:38,358 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:A 2024-11-14T09:44:38,358 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/1793b1e6aecb429fb92ab9fd9e3c36bf as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/1793b1e6aecb429fb92ab9fd9e3c36bf 2024-11-14T09:44:38,358 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:38,359 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:38,359 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/C is initiating minor compaction (all files) 2024-11-14T09:44:38,359 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/C in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:38,359 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/895bda11c8b14dbc856282d35350a27d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/2fce20641cb84105a5c00186c5aedac2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4ffd520b24154da182e74921a9f2fbef, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/25ad8e9a128c4d3b822fd212c4d35fa8] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=46.9 K 2024-11-14T09:44:38,360 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 895bda11c8b14dbc856282d35350a27d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577472947 2024-11-14T09:44:38,360 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fce20641cb84105a5c00186c5aedac2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731577472977 2024-11-14T09:44:38,361 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffd520b24154da182e74921a9f2fbef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577473313 2024-11-14T09:44:38,361 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ad8e9a128c4d3b822fd212c4d35fa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577475516 2024-11-14T09:44:38,363 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/B of ab93aec2a07aaf72eae598338b2d22a6 into 1793b1e6aecb429fb92ab9fd9e3c36bf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:38,363 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:38,363 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/B, priority=12, startTime=1731577477921; duration=0sec 2024-11-14T09:44:38,363 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:38,363 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:B 2024-11-14T09:44:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:44:38,370 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#C#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:38,371 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/ff5b2f318e254256a5e5bb683ec637ab is 50, key is test_row_0/C:col10/1731577475516/Put/seqid=0 2024-11-14T09:44:38,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742068_1244 (size=12139) 2024-11-14T09:44:38,407 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/31a34375a49a4b5e9411e38219b555ba 2024-11-14T09:44:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/04687f1c4c9849c2a22a1f83394f59c9 is 50, key is test_row_0/B:col10/1731577476666/Put/seqid=0 2024-11-14T09:44:38,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742069_1245 (size=12001) 2024-11-14T09:44:38,790 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/ff5b2f318e254256a5e5bb683ec637ab as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ff5b2f318e254256a5e5bb683ec637ab 2024-11-14T09:44:38,800 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/C of ab93aec2a07aaf72eae598338b2d22a6 into ff5b2f318e254256a5e5bb683ec637ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:38,801 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:38,801 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/C, priority=12, startTime=1731577477921; duration=0sec 2024-11-14T09:44:38,801 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:38,801 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:C 2024-11-14T09:44:38,822 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/04687f1c4c9849c2a22a1f83394f59c9 2024-11-14T09:44:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/e781e3a0907a40e99960c9ea7eed9019 is 50, key is test_row_0/C:col10/1731577476666/Put/seqid=0 2024-11-14T09:44:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:38,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:38,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742070_1246 (size=12001) 2024-11-14T09:44:38,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577538849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577478845, max timestamp=1731577478852Num uniques 859; 2024-11-14T09:44:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577478845, max timestamp=1731577478852Num uniques 859; 2024-11-14T09:44:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731577478845, max timestamp=1731577478852Num uniques 859; 2024-11-14T09:44:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:44:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577538870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577538870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577538872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577538873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577538960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577538980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577538980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577538980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577538980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577539170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577539190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577539190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577539190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577539190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,236 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/e781e3a0907a40e99960c9ea7eed9019 2024-11-14T09:44:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/31a34375a49a4b5e9411e38219b555ba as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/31a34375a49a4b5e9411e38219b555ba 2024-11-14T09:44:39,246 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/31a34375a49a4b5e9411e38219b555ba, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T09:44:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/04687f1c4c9849c2a22a1f83394f59c9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/04687f1c4c9849c2a22a1f83394f59c9 2024-11-14T09:44:39,250 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/04687f1c4c9849c2a22a1f83394f59c9, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T09:44:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/e781e3a0907a40e99960c9ea7eed9019 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/e781e3a0907a40e99960c9ea7eed9019 2024-11-14T09:44:39,255 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/e781e3a0907a40e99960c9ea7eed9019, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T09:44:39,256 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for ab93aec2a07aaf72eae598338b2d22a6 in 1260ms, sequenceid=78, compaction requested=false 2024-11-14T09:44:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-14T09:44:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-14T09:44:39,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T09:44:39,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-11-14T09:44:39,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.0320 sec 2024-11-14T09:44:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T09:44:39,380 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:39,380 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-14T09:44:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:44:39,382 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:39,382 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:39,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:39,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T09:44:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:39,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:39,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/d5ab9a681a434997811d0ddd0a7297f7 is 50, key is test_row_0/A:col10/1731577478869/Put/seqid=0 2024-11-14T09:44:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:44:39,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742071_1247 (size=14341) 2024-11-14T09:44:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577539501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577539503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577539504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577539505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577539505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577539610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577539621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577539621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577539621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577539621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:44:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577539820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577539830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577539830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577539830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577539830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:39,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/d5ab9a681a434997811d0ddd0a7297f7 2024-11-14T09:44:39,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/abb99721509a41f399aa67af26872526 is 50, key is test_row_0/B:col10/1731577478869/Put/seqid=0 2024-11-14T09:44:39,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742072_1248 (size=12001) 2024-11-14T09:44:39,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:39,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:44:40,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577540130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577540140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577540141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577540141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577540141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:40,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:40,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/abb99721509a41f399aa67af26872526 2024-11-14T09:44:40,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/18ca91becc0849adb10431600ab61d6b is 50, key is test_row_0/C:col10/1731577478869/Put/seqid=0 2024-11-14T09:44:40,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742073_1249 (size=12001) 2024-11-14T09:44:40,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:44:40,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:40,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577540641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577540650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577540650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577540650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577540650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:40,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/18ca91becc0849adb10431600ab61d6b 2024-11-14T09:44:40,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/d5ab9a681a434997811d0ddd0a7297f7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d5ab9a681a434997811d0ddd0a7297f7 2024-11-14T09:44:40,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d5ab9a681a434997811d0ddd0a7297f7, entries=200, sequenceid=116, filesize=14.0 K 2024-11-14T09:44:40,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/abb99721509a41f399aa67af26872526 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/abb99721509a41f399aa67af26872526 2024-11-14T09:44:40,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/abb99721509a41f399aa67af26872526, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:44:40,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/18ca91becc0849adb10431600ab61d6b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/18ca91becc0849adb10431600ab61d6b 2024-11-14T09:44:40,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/18ca91becc0849adb10431600ab61d6b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:44:40,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ab93aec2a07aaf72eae598338b2d22a6 in 1256ms, sequenceid=116, compaction requested=true 2024-11-14T09:44:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:40,740 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:40,740 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:40,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:40,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:40,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:40,741 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:40,742 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:40,742 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/A is initiating minor compaction (all files) 2024-11-14T09:44:40,742 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/B is initiating minor compaction (all files) 2024-11-14T09:44:40,742 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/A in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,742 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/B in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,742 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/1793b1e6aecb429fb92ab9fd9e3c36bf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/04687f1c4c9849c2a22a1f83394f59c9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/abb99721509a41f399aa67af26872526] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=35.3 K 2024-11-14T09:44:40,742 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/45ef2effe8fc4c53be226a2ea22466f3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/31a34375a49a4b5e9411e38219b555ba, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d5ab9a681a434997811d0ddd0a7297f7] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=37.6 K 2024-11-14T09:44:40,742 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45ef2effe8fc4c53be226a2ea22466f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577475516 2024-11-14T09:44:40,742 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 1793b1e6aecb429fb92ab9fd9e3c36bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577475516 2024-11-14T09:44:40,742 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31a34375a49a4b5e9411e38219b555ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731577476666 2024-11-14T09:44:40,743 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 04687f1c4c9849c2a22a1f83394f59c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731577476666 2024-11-14T09:44:40,743 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5ab9a681a434997811d0ddd0a7297f7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577478869 2024-11-14T09:44:40,743 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting abb99721509a41f399aa67af26872526, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577478869 2024-11-14T09:44:40,750 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:40,750 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/c542830ab612446db7fc248b063efaed is 50, key is test_row_0/A:col10/1731577478869/Put/seqid=0 2024-11-14T09:44:40,751 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#B#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:40,751 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d8a71157c1b747699a0aea3d70e193cf is 50, key is test_row_0/B:col10/1731577478869/Put/seqid=0 2024-11-14T09:44:40,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:44:40,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,757 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:44:40,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:40,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:40,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:40,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:40,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:40,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:40,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742074_1250 (size=12241) 2024-11-14T09:44:40,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/81cafc1872ba4ed0b5557bb83a6c560e is 50, key is test_row_0/A:col10/1731577479504/Put/seqid=0 2024-11-14T09:44:40,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742075_1251 (size=12241) 2024-11-14T09:44:40,783 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d8a71157c1b747699a0aea3d70e193cf as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d8a71157c1b747699a0aea3d70e193cf 2024-11-14T09:44:40,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742076_1252 (size=12001) 2024-11-14T09:44:40,789 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/B of ab93aec2a07aaf72eae598338b2d22a6 into d8a71157c1b747699a0aea3d70e193cf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:40,789 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:40,789 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/B, priority=13, startTime=1731577480740; duration=0sec 2024-11-14T09:44:40,789 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:40,789 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:B 2024-11-14T09:44:40,789 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:40,791 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:40,791 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/C is initiating minor compaction (all files) 2024-11-14T09:44:40,791 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/C in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:40,791 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ff5b2f318e254256a5e5bb683ec637ab, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/e781e3a0907a40e99960c9ea7eed9019, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/18ca91becc0849adb10431600ab61d6b] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=35.3 K 2024-11-14T09:44:40,791 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ff5b2f318e254256a5e5bb683ec637ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577475516 2024-11-14T09:44:40,791 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e781e3a0907a40e99960c9ea7eed9019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731577476666 2024-11-14T09:44:40,791 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ca91becc0849adb10431600ab61d6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577478869 2024-11-14T09:44:40,798 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:40,798 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/fa74787c732b492783d6b13d48eec3a5 is 50, key is test_row_0/C:col10/1731577478869/Put/seqid=0 2024-11-14T09:44:40,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742077_1253 (size=12241) 2024-11-14T09:44:41,181 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/c542830ab612446db7fc248b063efaed as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/c542830ab612446db7fc248b063efaed 2024-11-14T09:44:41,184 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/81cafc1872ba4ed0b5557bb83a6c560e 2024-11-14T09:44:41,186 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/A of ab93aec2a07aaf72eae598338b2d22a6 into c542830ab612446db7fc248b063efaed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:41,186 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:41,186 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/A, priority=13, startTime=1731577480740; duration=0sec 2024-11-14T09:44:41,187 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:41,187 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:A 2024-11-14T09:44:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/efc34917e2da4dd5a79b14b0980fc17d is 50, key is test_row_0/B:col10/1731577479504/Put/seqid=0 2024-11-14T09:44:41,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742078_1254 (size=12001) 2024-11-14T09:44:41,255 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/efc34917e2da4dd5a79b14b0980fc17d 2024-11-14T09:44:41,257 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/fa74787c732b492783d6b13d48eec3a5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/fa74787c732b492783d6b13d48eec3a5 2024-11-14T09:44:41,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/c2b36e05d434422281e37c6e4ba13607 is 50, key is test_row_0/C:col10/1731577479504/Put/seqid=0 2024-11-14T09:44:41,266 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/C of ab93aec2a07aaf72eae598338b2d22a6 into fa74787c732b492783d6b13d48eec3a5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:41,266 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:41,266 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/C, priority=13, startTime=1731577480741; duration=0sec 2024-11-14T09:44:41,266 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:41,266 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:C 2024-11-14T09:44:41,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742079_1255 (size=12001) 2024-11-14T09:44:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:44:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:41,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:41,675 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/c2b36e05d434422281e37c6e4ba13607 2024-11-14T09:44:41,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/81cafc1872ba4ed0b5557bb83a6c560e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/81cafc1872ba4ed0b5557bb83a6c560e 2024-11-14T09:44:41,684 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/81cafc1872ba4ed0b5557bb83a6c560e, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T09:44:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/efc34917e2da4dd5a79b14b0980fc17d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/efc34917e2da4dd5a79b14b0980fc17d 2024-11-14T09:44:41,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577481680, max timestamp=1731577481686Num uniques 859; 2024-11-14T09:44:41,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577481680, max timestamp=1731577481686Num uniques 859; 2024-11-14T09:44:41,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:41,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:41,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577481680, max timestamp=1731577481686Num uniques 859; 2024-11-14T09:44:41,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577541682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,691 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/efc34917e2da4dd5a79b14b0980fc17d, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T09:44:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/c2b36e05d434422281e37c6e4ba13607 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/c2b36e05d434422281e37c6e4ba13607 2024-11-14T09:44:41,696 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/c2b36e05d434422281e37c6e4ba13607, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T09:44:41,697 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=67.09 KB/68700 for ab93aec2a07aaf72eae598338b2d22a6 in 940ms, sequenceid=127, compaction requested=false 2024-11-14T09:44:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T09:44:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:41,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-11-14T09:44:41,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T09:44:41,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:41,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:41,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:41,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:41,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:41,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T09:44:41,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3160 sec 2024-11-14T09:44:41,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/70b36848ef774168a306310dd0f6f595 is 50, key is test_row_0/A:col10/1731577481689/Put/seqid=0 2024-11-14T09:44:41,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.3210 sec 2024-11-14T09:44:41,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742080_1256 (size=19321) 2024-11-14T09:44:41,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577541721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577541723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577541724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577541726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577541800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577541830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577541840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577541840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577541841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577542010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577542040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577542051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577542051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577542051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/70b36848ef774168a306310dd0f6f595 2024-11-14T09:44:42,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/890885b5ebc5443999d8aeedccc22011 is 50, key is test_row_0/B:col10/1731577481689/Put/seqid=0 2024-11-14T09:44:42,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742081_1257 (size=12151) 2024-11-14T09:44:42,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577542320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577542350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577542360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577542361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577542361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/890885b5ebc5443999d8aeedccc22011 2024-11-14T09:44:42,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/304a0cab025045218a361d13419229f2 is 50, key is test_row_0/C:col10/1731577481689/Put/seqid=0 2024-11-14T09:44:42,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742082_1258 (size=12151) 2024-11-14T09:44:42,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/304a0cab025045218a361d13419229f2 2024-11-14T09:44:42,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/70b36848ef774168a306310dd0f6f595 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/70b36848ef774168a306310dd0f6f595 2024-11-14T09:44:42,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/70b36848ef774168a306310dd0f6f595, entries=300, sequenceid=158, filesize=18.9 K 2024-11-14T09:44:42,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/890885b5ebc5443999d8aeedccc22011 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/890885b5ebc5443999d8aeedccc22011 2024-11-14T09:44:42,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/890885b5ebc5443999d8aeedccc22011, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T09:44:42,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/304a0cab025045218a361d13419229f2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/304a0cab025045218a361d13419229f2 2024-11-14T09:44:42,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/304a0cab025045218a361d13419229f2, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T09:44:42,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=107.34 KB/109920 for ab93aec2a07aaf72eae598338b2d22a6 in 853ms, sequenceid=158, compaction requested=true 2024-11-14T09:44:42,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:42,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:42,550 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:42,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:42,551 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:42,551 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:42,551 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43563 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:42,551 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/B is initiating minor compaction (all files) 2024-11-14T09:44:42,551 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/A is initiating minor compaction (all files) 2024-11-14T09:44:42,551 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/B in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:42,551 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/A in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:42,552 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d8a71157c1b747699a0aea3d70e193cf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/efc34917e2da4dd5a79b14b0980fc17d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/890885b5ebc5443999d8aeedccc22011] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=35.5 K 2024-11-14T09:44:42,552 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/c542830ab612446db7fc248b063efaed, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/81cafc1872ba4ed0b5557bb83a6c560e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/70b36848ef774168a306310dd0f6f595] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=42.5 K 2024-11-14T09:44:42,552 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a71157c1b747699a0aea3d70e193cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577478869 2024-11-14T09:44:42,552 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c542830ab612446db7fc248b063efaed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577478869 2024-11-14T09:44:42,552 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting efc34917e2da4dd5a79b14b0980fc17d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577479502 2024-11-14T09:44:42,552 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81cafc1872ba4ed0b5557bb83a6c560e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577479502 2024-11-14T09:44:42,553 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b36848ef774168a306310dd0f6f595, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731577481686 2024-11-14T09:44:42,553 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 890885b5ebc5443999d8aeedccc22011, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731577481689 2024-11-14T09:44:42,560 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:42,561 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:42,561 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/30e6a4f6c45348b1803bc77727cf6031 is 50, key is test_row_0/A:col10/1731577481689/Put/seqid=0 2024-11-14T09:44:42,561 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/c3c0f8b6397d473ab0e33027088dfba7 is 50, key is test_row_0/B:col10/1731577481689/Put/seqid=0 2024-11-14T09:44:42,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742083_1259 (size=12493) 2024-11-14T09:44:42,581 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/30e6a4f6c45348b1803bc77727cf6031 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/30e6a4f6c45348b1803bc77727cf6031 2024-11-14T09:44:42,588 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/A of ab93aec2a07aaf72eae598338b2d22a6 into 30e6a4f6c45348b1803bc77727cf6031(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:42,588 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:42,588 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/A, priority=13, startTime=1731577482550; duration=0sec 2024-11-14T09:44:42,588 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:42,588 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:A 2024-11-14T09:44:42,588 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:42,589 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:42,589 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/C is initiating minor compaction (all files) 2024-11-14T09:44:42,589 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/C in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:42,589 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/fa74787c732b492783d6b13d48eec3a5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/c2b36e05d434422281e37c6e4ba13607, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/304a0cab025045218a361d13419229f2] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=35.5 K 2024-11-14T09:44:42,590 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa74787c732b492783d6b13d48eec3a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577478869 2024-11-14T09:44:42,590 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b36e05d434422281e37c6e4ba13607, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577479502 2024-11-14T09:44:42,591 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 304a0cab025045218a361d13419229f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731577481689 2024-11-14T09:44:42,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742084_1260 (size=12493) 2024-11-14T09:44:42,599 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:42,600 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/ea6fdcae8a414dca80e0f464e2ea527e is 50, key is test_row_0/C:col10/1731577481689/Put/seqid=0 2024-11-14T09:44:42,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742085_1261 (size=12493) 2024-11-14T09:44:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:42,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:44:42,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:42,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:42,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:42,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:42,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:42,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:42,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dba85a3612434cdc9b6bb693f542b294 is 50, key is test_row_0/A:col10/1731577482831/Put/seqid=0 2024-11-14T09:44:42,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742086_1262 (size=12151) 2024-11-14T09:44:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577542864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577542865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577542870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577542880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577542880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577542970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577542970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:42,995 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/c3c0f8b6397d473ab0e33027088dfba7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/c3c0f8b6397d473ab0e33027088dfba7 2024-11-14T09:44:42,999 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/B of ab93aec2a07aaf72eae598338b2d22a6 into c3c0f8b6397d473ab0e33027088dfba7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:43,000 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:43,000 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/B, priority=13, startTime=1731577482550; duration=0sec 2024-11-14T09:44:43,000 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:43,000 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:B 2024-11-14T09:44:43,010 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/ea6fdcae8a414dca80e0f464e2ea527e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ea6fdcae8a414dca80e0f464e2ea527e 2024-11-14T09:44:43,015 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/C of ab93aec2a07aaf72eae598338b2d22a6 into ea6fdcae8a414dca80e0f464e2ea527e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:43,015 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:43,015 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/C, priority=13, startTime=1731577482551; duration=0sec 2024-11-14T09:44:43,015 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:43,015 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:C 2024-11-14T09:44:43,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577543181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:43,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577543181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:43,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dba85a3612434cdc9b6bb693f542b294 2024-11-14T09:44:43,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/111e40ca252441b3ad933d2c36b1f0a3 is 50, key is test_row_0/B:col10/1731577482831/Put/seqid=0 2024-11-14T09:44:43,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742087_1263 (size=12151) 2024-11-14T09:44:43,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:43,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577543490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577543490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:44:43,560 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:43,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T09:44:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:44:43,562 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:43,562 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:43,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:43,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/111e40ca252441b3ad933d2c36b1f0a3 2024-11-14T09:44:43,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/647068de1c19488e9464b92b1d015971 is 50, key is test_row_0/C:col10/1731577482831/Put/seqid=0 2024-11-14T09:44:43,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742088_1264 (size=12151) 2024-11-14T09:44:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:44:43,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:44:43,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:43,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:43,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:44:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:43,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:44:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577543890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577543890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577543891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:43,893 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:43,893 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:43,894 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577544001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577544001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:44,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:44:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:44,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:44,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/647068de1c19488e9464b92b1d015971 2024-11-14T09:44:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dba85a3612434cdc9b6bb693f542b294 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dba85a3612434cdc9b6bb693f542b294 2024-11-14T09:44:44,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dba85a3612434cdc9b6bb693f542b294, entries=150, sequenceid=179, filesize=11.9 K 2024-11-14T09:44:44,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/111e40ca252441b3ad933d2c36b1f0a3 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/111e40ca252441b3ad933d2c36b1f0a3 2024-11-14T09:44:44,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/111e40ca252441b3ad933d2c36b1f0a3, entries=150, sequenceid=179, filesize=11.9 K 2024-11-14T09:44:44,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/647068de1c19488e9464b92b1d015971 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/647068de1c19488e9464b92b1d015971 2024-11-14T09:44:44,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/647068de1c19488e9464b92b1d015971, entries=150, sequenceid=179, filesize=11.9 K 2024-11-14T09:44:44,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ab93aec2a07aaf72eae598338b2d22a6 in 1253ms, sequenceid=179, compaction requested=false 2024-11-14T09:44:44,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:44,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:44:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:44,173 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:44:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dcd0aa705d994effa9569e3507fb06c0 is 50, key is test_row_0/A:col10/1731577482854/Put/seqid=0 2024-11-14T09:44:44,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742089_1265 (size=12151) 2024-11-14T09:44:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:44:44,580 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dcd0aa705d994effa9569e3507fb06c0 2024-11-14T09:44:44,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/423442709e594521b91f07e7e1e303ef is 50, key is test_row_0/B:col10/1731577482854/Put/seqid=0 2024-11-14T09:44:44,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742090_1266 (size=12151) 2024-11-14T09:44:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:44:44,991 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/423442709e594521b91f07e7e1e303ef 2024-11-14T09:44:44,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/f686e16f2e83459480076ff90b5ee780 is 50, key is test_row_0/C:col10/1731577482854/Put/seqid=0 2024-11-14T09:44:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742091_1267 (size=12151) 2024-11-14T09:44:45,002 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/f686e16f2e83459480076ff90b5ee780 2024-11-14T09:44:45,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dcd0aa705d994effa9569e3507fb06c0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dcd0aa705d994effa9569e3507fb06c0 2024-11-14T09:44:45,010 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dcd0aa705d994effa9569e3507fb06c0, entries=150, sequenceid=193, filesize=11.9 K 2024-11-14T09:44:45,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/423442709e594521b91f07e7e1e303ef as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/423442709e594521b91f07e7e1e303ef 2024-11-14T09:44:45,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:45,015 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/423442709e594521b91f07e7e1e303ef, entries=150, sequenceid=193, filesize=11.9 K 2024-11-14T09:44:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/f686e16f2e83459480076ff90b5ee780 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f686e16f2e83459480076ff90b5ee780 2024-11-14T09:44:45,020 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f686e16f2e83459480076ff90b5ee780, entries=150, sequenceid=193, filesize=11.9 K 2024-11-14T09:44:45,021 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for ab93aec2a07aaf72eae598338b2d22a6 in 849ms, sequenceid=193, compaction requested=true 2024-11-14T09:44:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T09:44:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T09:44:45,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T09:44:45,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-14T09:44:45,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4640 sec 2024-11-14T09:44:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:45,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:44:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:45,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:45,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/efe55b41c9bc4085a5ac1f2900b49a60 is 50, key is test_row_0/A:col10/1731577485049/Put/seqid=0 2024-11-14T09:44:45,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742092_1268 (size=16931) 2024-11-14T09:44:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577485073, max timestamp=1731577485098Num uniques 859; 2024-11-14T09:44:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577485073, max timestamp=1731577485098Num uniques 859; 2024-11-14T09:44:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577485073, max timestamp=1731577485098Num uniques 859; 2024-11-14T09:44:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:44:45,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577545138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577545140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577545250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577545250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/efe55b41c9bc4085a5ac1f2900b49a60 2024-11-14T09:44:45,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577545461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577545461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d92e05a59dec4e3993e40234ffdddaa1 is 50, key is test_row_0/B:col10/1731577485049/Put/seqid=0 2024-11-14T09:44:45,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742093_1269 (size=12151) 2024-11-14T09:44:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:44:45,710 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:45,710 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T09:44:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:44:45,712 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:45,712 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:45,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577545770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577545770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:44:45,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:44:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:45,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:45,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d92e05a59dec4e3993e40234ffdddaa1 2024-11-14T09:44:45,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4668a00b929b486c8d1db2073c1bf7fd is 50, key is test_row_0/C:col10/1731577485049/Put/seqid=0 2024-11-14T09:44:45,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742094_1270 (size=12151) 2024-11-14T09:44:45,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577545900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,904 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577545910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577545910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:45,914 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:45,914 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:46,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:44:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:44:46,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:44:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577546281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577546281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:46,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4668a00b929b486c8d1db2073c1bf7fd 2024-11-14T09:44:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:46,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:44:46,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:46,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:44:46,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/efe55b41c9bc4085a5ac1f2900b49a60 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/efe55b41c9bc4085a5ac1f2900b49a60 2024-11-14T09:44:46,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/efe55b41c9bc4085a5ac1f2900b49a60, entries=250, sequenceid=204, filesize=16.5 K 2024-11-14T09:44:46,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d92e05a59dec4e3993e40234ffdddaa1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d92e05a59dec4e3993e40234ffdddaa1 2024-11-14T09:44:46,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d92e05a59dec4e3993e40234ffdddaa1, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T09:44:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4668a00b929b486c8d1db2073c1bf7fd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4668a00b929b486c8d1db2073c1bf7fd 2024-11-14T09:44:46,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4668a00b929b486c8d1db2073c1bf7fd, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T09:44:46,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ab93aec2a07aaf72eae598338b2d22a6 in 1330ms, sequenceid=204, compaction requested=true 2024-11-14T09:44:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:46,380 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:46,380 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:46,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/B is initiating minor compaction (all files) 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/A is initiating minor compaction (all files) 2024-11-14T09:44:46,381 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/B in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,381 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/A in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,381 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/c3c0f8b6397d473ab0e33027088dfba7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/111e40ca252441b3ad933d2c36b1f0a3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/423442709e594521b91f07e7e1e303ef, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d92e05a59dec4e3993e40234ffdddaa1] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=47.8 K 2024-11-14T09:44:46,381 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/30e6a4f6c45348b1803bc77727cf6031, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dba85a3612434cdc9b6bb693f542b294, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dcd0aa705d994effa9569e3507fb06c0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/efe55b41c9bc4085a5ac1f2900b49a60] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=52.5 K 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c3c0f8b6397d473ab0e33027088dfba7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731577481689 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30e6a4f6c45348b1803bc77727cf6031, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731577481689 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 111e40ca252441b3ad933d2c36b1f0a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731577481723 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dba85a3612434cdc9b6bb693f542b294, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731577481723 2024-11-14T09:44:46,381 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 423442709e594521b91f07e7e1e303ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731577482854 2024-11-14T09:44:46,382 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcd0aa705d994effa9569e3507fb06c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731577482854 2024-11-14T09:44:46,382 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d92e05a59dec4e3993e40234ffdddaa1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731577485045 2024-11-14T09:44:46,382 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting efe55b41c9bc4085a5ac1f2900b49a60, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731577485022 2024-11-14T09:44:46,390 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:46,390 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/2f64c5f839924e6f9b75e733293b92b0 is 50, key is test_row_0/B:col10/1731577485049/Put/seqid=0 2024-11-14T09:44:46,391 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#A#compaction#228 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:46,391 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/1a168ccdad0541b898deac3094f01224 is 50, key is test_row_0/A:col10/1731577485049/Put/seqid=0 2024-11-14T09:44:46,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742095_1271 (size=12629) 2024-11-14T09:44:46,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742096_1272 (size=12629) 2024-11-14T09:44:46,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:44:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,516 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-14T09:44:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:44:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/41d8394501ab494d9d76a4c45689aed7 is 50, key is test_row_0/A:col10/1731577485133/Put/seqid=0 2024-11-14T09:44:46,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742097_1273 (size=12151) 2024-11-14T09:44:46,525 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/41d8394501ab494d9d76a4c45689aed7 2024-11-14T09:44:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d5658608e2604dc3a7f9510e7384da5d is 50, key is test_row_0/B:col10/1731577485133/Put/seqid=0 2024-11-14T09:44:46,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742098_1274 (size=12151) 2024-11-14T09:44:46,802 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/2f64c5f839924e6f9b75e733293b92b0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2f64c5f839924e6f9b75e733293b92b0 2024-11-14T09:44:46,803 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/1a168ccdad0541b898deac3094f01224 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/1a168ccdad0541b898deac3094f01224 2024-11-14T09:44:46,807 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/B of ab93aec2a07aaf72eae598338b2d22a6 into 2f64c5f839924e6f9b75e733293b92b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:46,807 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:46,807 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/B, priority=12, startTime=1731577486380; duration=0sec 2024-11-14T09:44:46,807 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:46,807 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:B 2024-11-14T09:44:46,808 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:44:46,808 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/A of ab93aec2a07aaf72eae598338b2d22a6 into 1a168ccdad0541b898deac3094f01224(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:46,808 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:46,808 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/A, priority=12, startTime=1731577486380; duration=0sec 2024-11-14T09:44:46,808 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:46,808 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:A 2024-11-14T09:44:46,809 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:44:46,809 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/C is initiating minor compaction (all files) 2024-11-14T09:44:46,809 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/C in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:46,809 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ea6fdcae8a414dca80e0f464e2ea527e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/647068de1c19488e9464b92b1d015971, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f686e16f2e83459480076ff90b5ee780, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4668a00b929b486c8d1db2073c1bf7fd] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=47.8 K 2024-11-14T09:44:46,809 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6fdcae8a414dca80e0f464e2ea527e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731577481689 2024-11-14T09:44:46,810 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 647068de1c19488e9464b92b1d015971, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731577481723 2024-11-14T09:44:46,810 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f686e16f2e83459480076ff90b5ee780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731577482854 2024-11-14T09:44:46,810 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4668a00b929b486c8d1db2073c1bf7fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731577485045 2024-11-14T09:44:46,819 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:46,820 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/65c13382dddd432d864f583d88209f9f is 50, key is test_row_0/C:col10/1731577485049/Put/seqid=0 2024-11-14T09:44:46,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742099_1275 (size=12629) 2024-11-14T09:44:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:44:46,942 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d5658608e2604dc3a7f9510e7384da5d 2024-11-14T09:44:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/20b5ee7470df4e439d3e69022896e1d9 is 50, key is test_row_0/C:col10/1731577485133/Put/seqid=0 2024-11-14T09:44:46,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742100_1276 (size=12151) 2024-11-14T09:44:47,229 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/65c13382dddd432d864f583d88209f9f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/65c13382dddd432d864f583d88209f9f 2024-11-14T09:44:47,235 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/C of ab93aec2a07aaf72eae598338b2d22a6 into 65c13382dddd432d864f583d88209f9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:47,235 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:47,235 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/C, priority=12, startTime=1731577486380; duration=0sec 2024-11-14T09:44:47,235 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:47,235 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:C 2024-11-14T09:44:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:47,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577547307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577547307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,354 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/20b5ee7470df4e439d3e69022896e1d9 2024-11-14T09:44:47,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/41d8394501ab494d9d76a4c45689aed7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/41d8394501ab494d9d76a4c45689aed7 2024-11-14T09:44:47,361 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/41d8394501ab494d9d76a4c45689aed7, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T09:44:47,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/d5658608e2604dc3a7f9510e7384da5d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d5658608e2604dc3a7f9510e7384da5d 2024-11-14T09:44:47,366 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d5658608e2604dc3a7f9510e7384da5d, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T09:44:47,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/20b5ee7470df4e439d3e69022896e1d9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/20b5ee7470df4e439d3e69022896e1d9 2024-11-14T09:44:47,371 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/20b5ee7470df4e439d3e69022896e1d9, entries=150, sequenceid=240, filesize=11.9 K 2024-11-14T09:44:47,371 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for ab93aec2a07aaf72eae598338b2d22a6 in 855ms, sequenceid=240, compaction requested=false 2024-11-14T09:44:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T09:44:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T09:44:47,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T09:44:47,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6610 sec 2024-11-14T09:44:47,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.6650 sec 2024-11-14T09:44:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:47,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:44:47,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:47,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:47,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:47,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:47,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:47,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:47,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/0eeeca1de65541feb1c0ff47c521bbc3 is 50, key is test_row_0/A:col10/1731577487301/Put/seqid=0 2024-11-14T09:44:47,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742101_1277 (size=14541) 2024-11-14T09:44:47,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577547464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577547465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577547570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577547580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577547780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577547791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:47,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/0eeeca1de65541feb1c0ff47c521bbc3 2024-11-14T09:44:47,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/54e32252a946455fabd30e26354b4340 is 50, key is test_row_0/B:col10/1731577487301/Put/seqid=0 2024-11-14T09:44:47,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742102_1278 (size=12151) 2024-11-14T09:44:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:44:47,860 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:47,861 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T09:44:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:44:47,862 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:47,863 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:47,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:44:48,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:44:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577548090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577548100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:48,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:44:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:44:48,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/54e32252a946455fabd30e26354b4340 2024-11-14T09:44:48,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/f0afc671c512486aab8e981e2a78e69c is 50, key is test_row_0/C:col10/1731577487301/Put/seqid=0 2024-11-14T09:44:48,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742103_1279 (size=12151) 2024-11-14T09:44:48,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:44:48,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:44:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:44:48,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577548600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:48,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577548610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:48,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:48,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:48,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:44:48,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:48,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:48,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/f0afc671c512486aab8e981e2a78e69c 2024-11-14T09:44:48,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/0eeeca1de65541feb1c0ff47c521bbc3 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0eeeca1de65541feb1c0ff47c521bbc3 2024-11-14T09:44:48,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0eeeca1de65541feb1c0ff47c521bbc3, entries=200, sequenceid=256, filesize=14.2 K 2024-11-14T09:44:48,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/54e32252a946455fabd30e26354b4340 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/54e32252a946455fabd30e26354b4340 2024-11-14T09:44:48,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/54e32252a946455fabd30e26354b4340, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T09:44:48,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/f0afc671c512486aab8e981e2a78e69c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f0afc671c512486aab8e981e2a78e69c 2024-11-14T09:44:48,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f0afc671c512486aab8e981e2a78e69c, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T09:44:48,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ab93aec2a07aaf72eae598338b2d22a6 in 1247ms, sequenceid=256, compaction requested=true 2024-11-14T09:44:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:48,670 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:48,670 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:48,671 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:48,671 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:48,671 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/B is initiating minor compaction (all files) 2024-11-14T09:44:48,671 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/A is initiating minor compaction (all files) 2024-11-14T09:44:48,671 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/B in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,671 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/A in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,671 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/1a168ccdad0541b898deac3094f01224, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/41d8394501ab494d9d76a4c45689aed7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0eeeca1de65541feb1c0ff47c521bbc3] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=38.4 K 2024-11-14T09:44:48,671 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2f64c5f839924e6f9b75e733293b92b0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d5658608e2604dc3a7f9510e7384da5d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/54e32252a946455fabd30e26354b4340] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=36.1 K 2024-11-14T09:44:48,671 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f64c5f839924e6f9b75e733293b92b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731577485045 2024-11-14T09:44:48,672 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a168ccdad0541b898deac3094f01224, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731577485045 2024-11-14T09:44:48,672 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d5658608e2604dc3a7f9510e7384da5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731577485133 2024-11-14T09:44:48,672 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d8394501ab494d9d76a4c45689aed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731577485133 2024-11-14T09:44:48,672 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e32252a946455fabd30e26354b4340, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731577487301 2024-11-14T09:44:48,672 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eeeca1de65541feb1c0ff47c521bbc3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731577487301 2024-11-14T09:44:48,679 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:48,679 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/de791e7b45a54649837b90cf6e5d5dec is 50, key is test_row_0/B:col10/1731577487301/Put/seqid=0 2024-11-14T09:44:48,680 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#A#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:48,680 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/fd0830192d40479d8e88b237a34b76c9 is 50, key is test_row_0/A:col10/1731577487301/Put/seqid=0 2024-11-14T09:44:48,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742104_1280 (size=12731) 2024-11-14T09:44:48,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742105_1281 (size=12731) 2024-11-14T09:44:48,707 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/de791e7b45a54649837b90cf6e5d5dec as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/de791e7b45a54649837b90cf6e5d5dec 2024-11-14T09:44:48,714 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/B of ab93aec2a07aaf72eae598338b2d22a6 into de791e7b45a54649837b90cf6e5d5dec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:48,714 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:48,714 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/B, priority=13, startTime=1731577488670; duration=0sec 2024-11-14T09:44:48,714 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:48,714 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:B 2024-11-14T09:44:48,714 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:48,715 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:48,715 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/C is initiating minor compaction (all files) 2024-11-14T09:44:48,715 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/C in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,715 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/65c13382dddd432d864f583d88209f9f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/20b5ee7470df4e439d3e69022896e1d9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f0afc671c512486aab8e981e2a78e69c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=36.1 K 2024-11-14T09:44:48,716 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c13382dddd432d864f583d88209f9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731577485045 2024-11-14T09:44:48,716 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b5ee7470df4e439d3e69022896e1d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731577485133 2024-11-14T09:44:48,717 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f0afc671c512486aab8e981e2a78e69c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731577487301 2024-11-14T09:44:48,728 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#C#compaction#239 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:48,729 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/d3d01b34edee4a5b8fe75b4a4d8df162 is 50, key is test_row_0/C:col10/1731577487301/Put/seqid=0 2024-11-14T09:44:48,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742106_1282 (size=12731) 2024-11-14T09:44:48,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:44:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:48,779 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:44:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/06c9d567164f4d89874bc66222f5cadb is 50, key is test_row_0/A:col10/1731577487463/Put/seqid=0 2024-11-14T09:44:48,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742107_1283 (size=12301) 2024-11-14T09:44:48,788 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/06c9d567164f4d89874bc66222f5cadb 2024-11-14T09:44:48,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/9e5d07e25e3e41e3b0ecbde114486b34 is 50, key is test_row_0/B:col10/1731577487463/Put/seqid=0 2024-11-14T09:44:48,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742108_1284 (size=12301) 2024-11-14T09:44:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:44:49,102 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/fd0830192d40479d8e88b237a34b76c9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/fd0830192d40479d8e88b237a34b76c9 2024-11-14T09:44:49,106 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/A of ab93aec2a07aaf72eae598338b2d22a6 into fd0830192d40479d8e88b237a34b76c9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:49,106 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:49,106 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/A, priority=13, startTime=1731577488670; duration=0sec 2024-11-14T09:44:49,107 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:49,107 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:A 2024-11-14T09:44:49,155 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/d3d01b34edee4a5b8fe75b4a4d8df162 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/d3d01b34edee4a5b8fe75b4a4d8df162 2024-11-14T09:44:49,160 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/C of ab93aec2a07aaf72eae598338b2d22a6 into d3d01b34edee4a5b8fe75b4a4d8df162(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:49,160 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:49,160 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/C, priority=13, startTime=1731577488670; duration=0sec 2024-11-14T09:44:49,160 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:49,160 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:C 2024-11-14T09:44:49,219 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/9e5d07e25e3e41e3b0ecbde114486b34 2024-11-14T09:44:49,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/b7d83a1d0a4b46e1a18aec9cad34a4b2 is 50, key is test_row_0/C:col10/1731577487463/Put/seqid=0 2024-11-14T09:44:49,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742109_1285 (size=12301) 2024-11-14T09:44:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:49,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:49,637 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/b7d83a1d0a4b46e1a18aec9cad34a4b2 2024-11-14T09:44:49,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577549640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577549641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/06c9d567164f4d89874bc66222f5cadb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/06c9d567164f4d89874bc66222f5cadb 2024-11-14T09:44:49,655 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/06c9d567164f4d89874bc66222f5cadb, entries=150, sequenceid=276, filesize=12.0 K 2024-11-14T09:44:49,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/9e5d07e25e3e41e3b0ecbde114486b34 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/9e5d07e25e3e41e3b0ecbde114486b34 2024-11-14T09:44:49,659 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/9e5d07e25e3e41e3b0ecbde114486b34, entries=150, sequenceid=276, filesize=12.0 K 2024-11-14T09:44:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/b7d83a1d0a4b46e1a18aec9cad34a4b2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/b7d83a1d0a4b46e1a18aec9cad34a4b2 2024-11-14T09:44:49,663 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/b7d83a1d0a4b46e1a18aec9cad34a4b2, entries=150, sequenceid=276, filesize=12.0 K 2024-11-14T09:44:49,664 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ab93aec2a07aaf72eae598338b2d22a6 in 885ms, sequenceid=276, compaction requested=false 2024-11-14T09:44:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:49,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T09:44:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T09:44:49,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T09:44:49,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-14T09:44:49,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8060 sec 2024-11-14T09:44:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:49,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:44:49,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:49,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:49,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:49,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:49,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:49,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:49,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dc507290aea847448c5c442b7aec8b75 is 50, key is test_row_0/A:col10/1731577489752/Put/seqid=0 2024-11-14T09:44:49,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742110_1286 (size=17181) 2024-11-14T09:44:49,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577549784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577549787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577549890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577549900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55508 deadline: 1731577549931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,933 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55500 deadline: 1731577549940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,942 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55502 deadline: 1731577549960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:49,963 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ab93aec2a07aaf72eae598338b2d22a6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:44:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:44:50,010 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:50,010 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T09:44:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:44:50,012 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:50,012 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:50,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:50,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577550100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:50,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577550111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:44:50,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:44:50,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:50,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dc507290aea847448c5c442b7aec8b75 2024-11-14T09:44:50,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/bd608439a65a4f94b4f71ee2f9a2969d is 50, key is test_row_0/B:col10/1731577489752/Put/seqid=0 2024-11-14T09:44:50,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742111_1287 (size=12301) 2024-11-14T09:44:50,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:44:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:44:50,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577550410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:50,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:50,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577550420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:50,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:44:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/bd608439a65a4f94b4f71ee2f9a2969d 2024-11-14T09:44:50,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/98d74d7d0bc64e0d99b59ca85862eb86 is 50, key is test_row_0/C:col10/1731577489752/Put/seqid=0 2024-11-14T09:44:50,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742112_1288 (size=12301) 2024-11-14T09:44:50,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:44:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:44:50,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:44:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577550920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:50,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:44:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:50,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:50,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577550930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:50,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/98d74d7d0bc64e0d99b59ca85862eb86 2024-11-14T09:44:50,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/dc507290aea847448c5c442b7aec8b75 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dc507290aea847448c5c442b7aec8b75 2024-11-14T09:44:50,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dc507290aea847448c5c442b7aec8b75, entries=250, sequenceid=293, filesize=16.8 K 2024-11-14T09:44:50,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/bd608439a65a4f94b4f71ee2f9a2969d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bd608439a65a4f94b4f71ee2f9a2969d 2024-11-14T09:44:51,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bd608439a65a4f94b4f71ee2f9a2969d, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T09:44:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/98d74d7d0bc64e0d99b59ca85862eb86 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/98d74d7d0bc64e0d99b59ca85862eb86 2024-11-14T09:44:51,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/98d74d7d0bc64e0d99b59ca85862eb86, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T09:44:51,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ab93aec2a07aaf72eae598338b2d22a6 in 1256ms, sequenceid=293, compaction requested=true 2024-11-14T09:44:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:51,009 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:51,009 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:51,010 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:51,010 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/A is initiating minor compaction (all files) 2024-11-14T09:44:51,010 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/A in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:51,010 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/fd0830192d40479d8e88b237a34b76c9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/06c9d567164f4d89874bc66222f5cadb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dc507290aea847448c5c442b7aec8b75] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=41.2 K 2024-11-14T09:44:51,010 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:51,010 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/B is initiating minor compaction (all files) 2024-11-14T09:44:51,010 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/B in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:51,010 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/de791e7b45a54649837b90cf6e5d5dec, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/9e5d07e25e3e41e3b0ecbde114486b34, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bd608439a65a4f94b4f71ee2f9a2969d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=36.5 K 2024-11-14T09:44:51,010 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd0830192d40479d8e88b237a34b76c9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731577487301 2024-11-14T09:44:51,011 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06c9d567164f4d89874bc66222f5cadb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731577487453 2024-11-14T09:44:51,011 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting de791e7b45a54649837b90cf6e5d5dec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731577487301 2024-11-14T09:44:51,011 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e5d07e25e3e41e3b0ecbde114486b34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731577487453 2024-11-14T09:44:51,011 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc507290aea847448c5c442b7aec8b75, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731577489631 2024-11-14T09:44:51,011 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting bd608439a65a4f94b4f71ee2f9a2969d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731577489631 2024-11-14T09:44:51,026 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:51,026 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:51,027 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/5ddf62e649944ad297bdd4818959fb8d is 50, key is test_row_0/A:col10/1731577489752/Put/seqid=0 2024-11-14T09:44:51,027 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/64605091a20e448cbf757f4b472edfa5 is 50, key is test_row_0/B:col10/1731577489752/Put/seqid=0 2024-11-14T09:44:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742113_1289 (size=12983) 2024-11-14T09:44:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742114_1290 (size=12983) 2024-11-14T09:44:51,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:44:51,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:51,084 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:44:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:51,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/58f89c3729bc4858bcb09a59fb91afec is 50, key is test_row_0/A:col10/1731577489777/Put/seqid=0 2024-11-14T09:44:51,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742115_1291 (size=12301) 2024-11-14T09:44:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:44:51,490 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/5ddf62e649944ad297bdd4818959fb8d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5ddf62e649944ad297bdd4818959fb8d 2024-11-14T09:44:51,492 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/64605091a20e448cbf757f4b472edfa5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/64605091a20e448cbf757f4b472edfa5 2024-11-14T09:44:51,494 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/58f89c3729bc4858bcb09a59fb91afec 2024-11-14T09:44:51,496 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/A of ab93aec2a07aaf72eae598338b2d22a6 into 5ddf62e649944ad297bdd4818959fb8d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:51,496 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:51,496 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/A, priority=13, startTime=1731577491009; duration=0sec 2024-11-14T09:44:51,496 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:51,496 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:A 2024-11-14T09:44:51,496 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:51,497 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:51,498 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/C is initiating minor compaction (all files) 2024-11-14T09:44:51,498 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/C in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:51,498 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/d3d01b34edee4a5b8fe75b4a4d8df162, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/b7d83a1d0a4b46e1a18aec9cad34a4b2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/98d74d7d0bc64e0d99b59ca85862eb86] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=36.5 K 2024-11-14T09:44:51,498 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/B of ab93aec2a07aaf72eae598338b2d22a6 into 64605091a20e448cbf757f4b472edfa5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:51,499 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:51,499 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/B, priority=13, startTime=1731577491009; duration=0sec 2024-11-14T09:44:51,499 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d01b34edee4a5b8fe75b4a4d8df162, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731577487301 2024-11-14T09:44:51,499 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d83a1d0a4b46e1a18aec9cad34a4b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731577487453 2024-11-14T09:44:51,499 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d74d7d0bc64e0d99b59ca85862eb86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731577489631 2024-11-14T09:44:51,504 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:51,504 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:B 2024-11-14T09:44:51,508 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:51,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/b097b8e9e79043e4a3245e7e6f6d9759 is 50, key is test_row_0/B:col10/1731577489777/Put/seqid=0 2024-11-14T09:44:51,508 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/95b5eea5a3b04e9389ab3f55f7d5569c is 50, key is test_row_0/C:col10/1731577489752/Put/seqid=0 2024-11-14T09:44:51,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742116_1292 (size=12301) 2024-11-14T09:44:51,530 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/b097b8e9e79043e4a3245e7e6f6d9759 2024-11-14T09:44:51,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742117_1293 (size=12983) 2024-11-14T09:44:51,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/31fa625d08a44392be93d93326db4c78 is 50, key is test_row_0/C:col10/1731577489777/Put/seqid=0 2024-11-14T09:44:51,547 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/95b5eea5a3b04e9389ab3f55f7d5569c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/95b5eea5a3b04e9389ab3f55f7d5569c 2024-11-14T09:44:51,551 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/C of ab93aec2a07aaf72eae598338b2d22a6 into 95b5eea5a3b04e9389ab3f55f7d5569c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:51,552 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:51,552 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/C, priority=13, startTime=1731577491009; duration=0sec 2024-11-14T09:44:51,552 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:51,552 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:C 2024-11-14T09:44:51,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742118_1294 (size=12301) 2024-11-14T09:44:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:51,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:51,955 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/31fa625d08a44392be93d93326db4c78 2024-11-14T09:44:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577551954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/58f89c3729bc4858bcb09a59fb91afec as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/58f89c3729bc4858bcb09a59fb91afec 2024-11-14T09:44:51,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577551956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:51,963 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/58f89c3729bc4858bcb09a59fb91afec, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T09:44:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/b097b8e9e79043e4a3245e7e6f6d9759 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b097b8e9e79043e4a3245e7e6f6d9759 2024-11-14T09:44:51,967 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b097b8e9e79043e4a3245e7e6f6d9759, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T09:44:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/31fa625d08a44392be93d93326db4c78 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/31fa625d08a44392be93d93326db4c78 2024-11-14T09:44:51,971 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/31fa625d08a44392be93d93326db4c78, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T09:44:51,972 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ab93aec2a07aaf72eae598338b2d22a6 in 889ms, sequenceid=310, compaction requested=false 2024-11-14T09:44:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T09:44:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T09:44:51,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T09:44:51,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9600 sec 2024-11-14T09:44:51,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9650 sec 2024-11-14T09:44:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:52,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:44:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:52,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:52,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/f0bbed1456854f43b45e2067f9d7dfe7 is 50, key is test_row_0/A:col10/1731577491954/Put/seqid=0 2024-11-14T09:44:52,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742119_1295 (size=14741) 2024-11-14T09:44:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577552089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577552092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:44:52,160 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:52,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:44:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T09:44:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:44:52,162 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:44:52,163 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:44:52,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:44:52,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577552200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577552200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:44:52,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577552410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577552410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/f0bbed1456854f43b45e2067f9d7dfe7 2024-11-14T09:44:52,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/58c0bdc7f1e5405183a6fa36de19cc15 is 50, key is test_row_0/B:col10/1731577491954/Put/seqid=0 2024-11-14T09:44:52,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742120_1296 (size=12301) 2024-11-14T09:44:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:44:52,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577552720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577552720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:52,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:52,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:52,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:44:52,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/58c0bdc7f1e5405183a6fa36de19cc15 2024-11-14T09:44:52,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4d3bf7d1845644e7b0a429b5903637f7 is 50, key is test_row_0/C:col10/1731577491954/Put/seqid=0 2024-11-14T09:44:52,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742121_1297 (size=12301) 2024-11-14T09:44:52,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:52,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:52,935 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-14T09:44:52,935 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:52,935 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,935 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:52,936 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-14T09:44:52,936 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:52,936 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,936 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:52,938 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-14T09:44:52,938 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:52,938 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,938 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:52,939 INFO [Thread-1087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1087. 2024-11-14T09:44:52,939 DEBUG [Thread-1087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:52,939 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,939 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:52,940 INFO [Thread-1085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1085. 2024-11-14T09:44:52,940 DEBUG [Thread-1085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:52,940 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,940 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:52,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:53,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:44:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55424 deadline: 1731577553230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1731577553231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:44:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:44:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:44:53,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:44:53,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4d3bf7d1845644e7b0a429b5903637f7 2024-11-14T09:44:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:44:53,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/f0bbed1456854f43b45e2067f9d7dfe7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f0bbed1456854f43b45e2067f9d7dfe7 2024-11-14T09:44:53,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f0bbed1456854f43b45e2067f9d7dfe7, entries=200, sequenceid=329, filesize=14.4 K 2024-11-14T09:44:53,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/58c0bdc7f1e5405183a6fa36de19cc15 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/58c0bdc7f1e5405183a6fa36de19cc15 2024-11-14T09:44:53,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/58c0bdc7f1e5405183a6fa36de19cc15, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T09:44:53,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/4d3bf7d1845644e7b0a429b5903637f7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4d3bf7d1845644e7b0a429b5903637f7 2024-11-14T09:44:53,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4d3bf7d1845644e7b0a429b5903637f7, entries=150, sequenceid=329, filesize=12.0 K 2024-11-14T09:44:53,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ab93aec2a07aaf72eae598338b2d22a6 in 1262ms, sequenceid=329, compaction requested=true 2024-11-14T09:44:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:44:53,324 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:44:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ab93aec2a07aaf72eae598338b2d22a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:44:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:44:53,324 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:53,325 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:53,326 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/A is initiating minor compaction (all files) 2024-11-14T09:44:53,326 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:53,326 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/A in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,326 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/B is initiating minor compaction (all files) 2024-11-14T09:44:53,326 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/B in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,326 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5ddf62e649944ad297bdd4818959fb8d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/58f89c3729bc4858bcb09a59fb91afec, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f0bbed1456854f43b45e2067f9d7dfe7] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=39.1 K 2024-11-14T09:44:53,326 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/64605091a20e448cbf757f4b472edfa5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b097b8e9e79043e4a3245e7e6f6d9759, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/58c0bdc7f1e5405183a6fa36de19cc15] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=36.7 K 2024-11-14T09:44:53,326 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 64605091a20e448cbf757f4b472edfa5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731577489631 2024-11-14T09:44:53,326 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ddf62e649944ad297bdd4818959fb8d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731577489631 2024-11-14T09:44:53,327 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b097b8e9e79043e4a3245e7e6f6d9759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731577489777 2024-11-14T09:44:53,327 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58f89c3729bc4858bcb09a59fb91afec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731577489777 2024-11-14T09:44:53,327 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c0bdc7f1e5405183a6fa36de19cc15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731577491948 2024-11-14T09:44:53,327 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0bbed1456854f43b45e2067f9d7dfe7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731577491948 2024-11-14T09:44:53,336 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:53,336 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:53,336 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/d47a30c3840c4da38865bdbfc1ec9f4c is 50, key is test_row_0/A:col10/1731577491954/Put/seqid=0 2024-11-14T09:44:53,336 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/2b91fad9a0e142aa8c3c2b65f8b2c7f2 is 50, key is test_row_0/B:col10/1731577491954/Put/seqid=0 2024-11-14T09:44:53,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742123_1299 (size=13085) 2024-11-14T09:44:53,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742122_1298 (size=13085) 2024-11-14T09:44:53,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:44:53,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,391 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:44:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:44:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:44:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:44:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:44:53,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/f2a7126a2180406fa79f88b89227793a is 50, key is test_row_0/A:col10/1731577492076/Put/seqid=0 2024-11-14T09:44:53,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742124_1300 (size=12301) 2024-11-14T09:44:53,752 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/d47a30c3840c4da38865bdbfc1ec9f4c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d47a30c3840c4da38865bdbfc1ec9f4c 2024-11-14T09:44:53,752 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/2b91fad9a0e142aa8c3c2b65f8b2c7f2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2b91fad9a0e142aa8c3c2b65f8b2c7f2 2024-11-14T09:44:53,756 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/B of ab93aec2a07aaf72eae598338b2d22a6 into 2b91fad9a0e142aa8c3c2b65f8b2c7f2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:53,756 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/A of ab93aec2a07aaf72eae598338b2d22a6 into d47a30c3840c4da38865bdbfc1ec9f4c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:53,756 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:53,756 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:53,756 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/A, priority=13, startTime=1731577493324; duration=0sec 2024-11-14T09:44:53,756 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/B, priority=13, startTime=1731577493324; duration=0sec 2024-11-14T09:44:53,756 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:53,756 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:44:53,756 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:B 2024-11-14T09:44:53,756 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:A 2024-11-14T09:44:53,757 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:44:53,757 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:44:53,757 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ab93aec2a07aaf72eae598338b2d22a6/C is initiating minor compaction (all files) 2024-11-14T09:44:53,758 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ab93aec2a07aaf72eae598338b2d22a6/C in TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:53,758 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/95b5eea5a3b04e9389ab3f55f7d5569c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/31fa625d08a44392be93d93326db4c78, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4d3bf7d1845644e7b0a429b5903637f7] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp, totalSize=36.7 K 2024-11-14T09:44:53,758 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 95b5eea5a3b04e9389ab3f55f7d5569c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731577489631 2024-11-14T09:44:53,758 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fa625d08a44392be93d93326db4c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731577489777 2024-11-14T09:44:53,758 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3bf7d1845644e7b0a429b5903637f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731577491948 2024-11-14T09:44:53,765 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ab93aec2a07aaf72eae598338b2d22a6#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:44:53,765 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/0f31b50c2a6748d2b53539ecff7c0e85 is 50, key is test_row_0/C:col10/1731577491954/Put/seqid=0 2024-11-14T09:44:53,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742125_1301 (size=13085) 2024-11-14T09:44:53,803 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/f2a7126a2180406fa79f88b89227793a 2024-11-14T09:44:53,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/3bb9a800d35f429aa3c1ccfda550b371 is 50, key is test_row_0/B:col10/1731577492076/Put/seqid=0 2024-11-14T09:44:53,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742126_1302 (size=12301) 2024-11-14T09:44:54,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/0f31b50c2a6748d2b53539ecff7c0e85 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/0f31b50c2a6748d2b53539ecff7c0e85 2024-11-14T09:44:54,190 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ab93aec2a07aaf72eae598338b2d22a6/C of ab93aec2a07aaf72eae598338b2d22a6 into 0f31b50c2a6748d2b53539ecff7c0e85(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:44:54,190 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:54,190 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., storeName=ab93aec2a07aaf72eae598338b2d22a6/C, priority=13, startTime=1731577493324; duration=0sec 2024-11-14T09:44:54,190 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:44:54,190 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ab93aec2a07aaf72eae598338b2d22a6:C 2024-11-14T09:44:54,214 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/3bb9a800d35f429aa3c1ccfda550b371 2024-11-14T09:44:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/a7ef213100474de4879d64401581511d is 50, key is test_row_0/C:col10/1731577492076/Put/seqid=0 2024-11-14T09:44:54,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742127_1303 (size=12301) 2024-11-14T09:44:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:44:54,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. as already flushing 2024-11-14T09:44:54,243 INFO [Thread-1078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1078. 2024-11-14T09:44:54,243 INFO [Thread-1076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1076. 2024-11-14T09:44:54,243 DEBUG [Thread-1078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:54,244 DEBUG [Thread-1076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:54,244 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:54,244 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:54,244 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:54,244 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:54,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:54,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:44:54,632 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/a7ef213100474de4879d64401581511d 2024-11-14T09:44:54,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/f2a7126a2180406fa79f88b89227793a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f2a7126a2180406fa79f88b89227793a 2024-11-14T09:44:54,648 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f2a7126a2180406fa79f88b89227793a, entries=150, sequenceid=345, filesize=12.0 K 2024-11-14T09:44:54,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/3bb9a800d35f429aa3c1ccfda550b371 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/3bb9a800d35f429aa3c1ccfda550b371 2024-11-14T09:44:54,654 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/3bb9a800d35f429aa3c1ccfda550b371, entries=150, sequenceid=345, filesize=12.0 K 2024-11-14T09:44:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/a7ef213100474de4879d64401581511d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/a7ef213100474de4879d64401581511d 2024-11-14T09:44:54,660 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/a7ef213100474de4879d64401581511d, entries=150, sequenceid=345, filesize=12.0 K 2024-11-14T09:44:54,661 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for ab93aec2a07aaf72eae598338b2d22a6 in 1271ms, sequenceid=345, compaction requested=false 2024-11-14T09:44:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ab93aec2a07aaf72eae598338b2d22a6: 2024-11-14T09:44:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:44:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T09:44:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T09:44:54,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T09:44:54,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4990 sec 2024-11-14T09:44:54,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.5030 sec 2024-11-14T09:44:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:44:56,321 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:44:56,642 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:44:59,955 INFO [Thread-1070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1070. 2024-11-14T09:44:59,955 DEBUG [Thread-1070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:59,955 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:59,955 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:59,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:44:59,972 INFO [Thread-1072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1072. 2024-11-14T09:44:59,972 DEBUG [Thread-1072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:44:59,972 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:59,972 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:44:59,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:00,055 INFO [Thread-1074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1074. 2024-11-14T09:45:00,055 DEBUG [Thread-1074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:00,055 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:00,055 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:00,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3796 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3597 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3604 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3789 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3618 2024-11-14T09:45:00,056 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:45:00,056 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:45:00,057 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:45:00,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:00,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:00,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:00,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:45:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:45:00,064 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577500064"}]},"ts":"1731577500064"} 2024-11-14T09:45:00,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:45:00,066 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:45:00,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:45:00,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, UNASSIGN}] 2024-11-14T09:45:00,070 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, UNASSIGN 2024-11-14T09:45:00,070 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=ab93aec2a07aaf72eae598338b2d22a6, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:00,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, UNASSIGN because future has completed 2024-11-14T09:45:00,072 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:45:00,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:45:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:45:00,226 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:45:00,226 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:45:00,226 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing ab93aec2a07aaf72eae598338b2d22a6, disabling compactions & flushes 2024-11-14T09:45:00,226 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:45:00,226 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:45:00,226 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. after waiting 0 ms 2024-11-14T09:45:00,226 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:45:00,226 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing ab93aec2a07aaf72eae598338b2d22a6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T09:45:00,227 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=A 2024-11-14T09:45:00,227 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:00,227 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=B 2024-11-14T09:45:00,227 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:00,227 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ab93aec2a07aaf72eae598338b2d22a6, store=C 2024-11-14T09:45:00,227 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:00,234 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/a435878535a44ef7b90aa4989230ef26 is 50, key is test_row_0/A:col10/1731577494241/Put/seqid=0 2024-11-14T09:45:00,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742128_1304 (size=12301) 2024-11-14T09:45:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:45:00,642 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/a435878535a44ef7b90aa4989230ef26 2024-11-14T09:45:00,657 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/b2857c14a7b0404aa2b753b8ff0b20b9 is 50, key is test_row_0/B:col10/1731577494241/Put/seqid=0 2024-11-14T09:45:00,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742129_1305 (size=12301) 2024-11-14T09:45:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:45:01,062 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/b2857c14a7b0404aa2b753b8ff0b20b9 2024-11-14T09:45:01,074 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/74838d54d0fa456a8040c31b9b53c09e is 50, key is test_row_0/C:col10/1731577494241/Put/seqid=0 2024-11-14T09:45:01,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742130_1306 (size=12301) 2024-11-14T09:45:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:45:01,481 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/74838d54d0fa456a8040c31b9b53c09e 2024-11-14T09:45:01,490 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/A/a435878535a44ef7b90aa4989230ef26 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/a435878535a44ef7b90aa4989230ef26 2024-11-14T09:45:01,496 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/a435878535a44ef7b90aa4989230ef26, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T09:45:01,497 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/B/b2857c14a7b0404aa2b753b8ff0b20b9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b2857c14a7b0404aa2b753b8ff0b20b9 2024-11-14T09:45:01,501 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b2857c14a7b0404aa2b753b8ff0b20b9, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T09:45:01,502 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/.tmp/C/74838d54d0fa456a8040c31b9b53c09e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/74838d54d0fa456a8040c31b9b53c09e 2024-11-14T09:45:01,506 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/74838d54d0fa456a8040c31b9b53c09e, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T09:45:01,507 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ab93aec2a07aaf72eae598338b2d22a6 in 1281ms, sequenceid=356, compaction requested=true 2024-11-14T09:45:01,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5a421938c92f416fa7b0fc9534b0b8c4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0e4323b7aa4a466da6adc644cb920738, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/4c4cde2d77df49a9a14f291e09046b43, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/edd105e13aa440e3808816672349b892, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/45ef2effe8fc4c53be226a2ea22466f3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/31a34375a49a4b5e9411e38219b555ba, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d5ab9a681a434997811d0ddd0a7297f7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/c542830ab612446db7fc248b063efaed, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/81cafc1872ba4ed0b5557bb83a6c560e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/70b36848ef774168a306310dd0f6f595, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/30e6a4f6c45348b1803bc77727cf6031, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dba85a3612434cdc9b6bb693f542b294, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dcd0aa705d994effa9569e3507fb06c0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/efe55b41c9bc4085a5ac1f2900b49a60, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/1a168ccdad0541b898deac3094f01224, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/41d8394501ab494d9d76a4c45689aed7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0eeeca1de65541feb1c0ff47c521bbc3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/fd0830192d40479d8e88b237a34b76c9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/06c9d567164f4d89874bc66222f5cadb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dc507290aea847448c5c442b7aec8b75, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5ddf62e649944ad297bdd4818959fb8d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/58f89c3729bc4858bcb09a59fb91afec, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f0bbed1456854f43b45e2067f9d7dfe7] to archive 2024-11-14T09:45:01,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:45:01,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5a421938c92f416fa7b0fc9534b0b8c4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5a421938c92f416fa7b0fc9534b0b8c4 2024-11-14T09:45:01,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0e4323b7aa4a466da6adc644cb920738 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0e4323b7aa4a466da6adc644cb920738 2024-11-14T09:45:01,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/4c4cde2d77df49a9a14f291e09046b43 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/4c4cde2d77df49a9a14f291e09046b43 2024-11-14T09:45:01,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/edd105e13aa440e3808816672349b892 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/edd105e13aa440e3808816672349b892 2024-11-14T09:45:01,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/45ef2effe8fc4c53be226a2ea22466f3 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/45ef2effe8fc4c53be226a2ea22466f3 2024-11-14T09:45:01,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/31a34375a49a4b5e9411e38219b555ba to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/31a34375a49a4b5e9411e38219b555ba 2024-11-14T09:45:01,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d5ab9a681a434997811d0ddd0a7297f7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d5ab9a681a434997811d0ddd0a7297f7 2024-11-14T09:45:01,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/c542830ab612446db7fc248b063efaed to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/c542830ab612446db7fc248b063efaed 2024-11-14T09:45:01,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/81cafc1872ba4ed0b5557bb83a6c560e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/81cafc1872ba4ed0b5557bb83a6c560e 2024-11-14T09:45:01,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/70b36848ef774168a306310dd0f6f595 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/70b36848ef774168a306310dd0f6f595 2024-11-14T09:45:01,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/30e6a4f6c45348b1803bc77727cf6031 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/30e6a4f6c45348b1803bc77727cf6031 2024-11-14T09:45:01,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dba85a3612434cdc9b6bb693f542b294 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dba85a3612434cdc9b6bb693f542b294 2024-11-14T09:45:01,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dcd0aa705d994effa9569e3507fb06c0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dcd0aa705d994effa9569e3507fb06c0 2024-11-14T09:45:01,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/efe55b41c9bc4085a5ac1f2900b49a60 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/efe55b41c9bc4085a5ac1f2900b49a60 2024-11-14T09:45:01,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/1a168ccdad0541b898deac3094f01224 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/1a168ccdad0541b898deac3094f01224 2024-11-14T09:45:01,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/41d8394501ab494d9d76a4c45689aed7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/41d8394501ab494d9d76a4c45689aed7 2024-11-14T09:45:01,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0eeeca1de65541feb1c0ff47c521bbc3 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/0eeeca1de65541feb1c0ff47c521bbc3 2024-11-14T09:45:01,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/fd0830192d40479d8e88b237a34b76c9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/fd0830192d40479d8e88b237a34b76c9 2024-11-14T09:45:01,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/06c9d567164f4d89874bc66222f5cadb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/06c9d567164f4d89874bc66222f5cadb 2024-11-14T09:45:01,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dc507290aea847448c5c442b7aec8b75 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/dc507290aea847448c5c442b7aec8b75 2024-11-14T09:45:01,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5ddf62e649944ad297bdd4818959fb8d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/5ddf62e649944ad297bdd4818959fb8d 2024-11-14T09:45:01,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/58f89c3729bc4858bcb09a59fb91afec to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/58f89c3729bc4858bcb09a59fb91afec 2024-11-14T09:45:01,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f0bbed1456854f43b45e2067f9d7dfe7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f0bbed1456854f43b45e2067f9d7dfe7 2024-11-14T09:45:01,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bfe18b16ae204585bb55d4b48b6f32b7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/52b9841b5533455090e943d85e49c5c4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/010d031a4bf049eaba7172f44d0f3909, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/1793b1e6aecb429fb92ab9fd9e3c36bf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/7cd6b7b7a14b4f19b6d5845ce65d2623, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/04687f1c4c9849c2a22a1f83394f59c9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d8a71157c1b747699a0aea3d70e193cf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/abb99721509a41f399aa67af26872526, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/efc34917e2da4dd5a79b14b0980fc17d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/c3c0f8b6397d473ab0e33027088dfba7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/890885b5ebc5443999d8aeedccc22011, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/111e40ca252441b3ad933d2c36b1f0a3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/423442709e594521b91f07e7e1e303ef, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2f64c5f839924e6f9b75e733293b92b0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d92e05a59dec4e3993e40234ffdddaa1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d5658608e2604dc3a7f9510e7384da5d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/de791e7b45a54649837b90cf6e5d5dec, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/54e32252a946455fabd30e26354b4340, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/9e5d07e25e3e41e3b0ecbde114486b34, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/64605091a20e448cbf757f4b472edfa5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bd608439a65a4f94b4f71ee2f9a2969d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b097b8e9e79043e4a3245e7e6f6d9759, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/58c0bdc7f1e5405183a6fa36de19cc15] to archive 2024-11-14T09:45:01,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:45:01,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bfe18b16ae204585bb55d4b48b6f32b7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bfe18b16ae204585bb55d4b48b6f32b7 2024-11-14T09:45:01,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/52b9841b5533455090e943d85e49c5c4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/52b9841b5533455090e943d85e49c5c4 2024-11-14T09:45:01,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/010d031a4bf049eaba7172f44d0f3909 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/010d031a4bf049eaba7172f44d0f3909 2024-11-14T09:45:01,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/1793b1e6aecb429fb92ab9fd9e3c36bf to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/1793b1e6aecb429fb92ab9fd9e3c36bf 2024-11-14T09:45:01,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/7cd6b7b7a14b4f19b6d5845ce65d2623 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/7cd6b7b7a14b4f19b6d5845ce65d2623 2024-11-14T09:45:01,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/04687f1c4c9849c2a22a1f83394f59c9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/04687f1c4c9849c2a22a1f83394f59c9 2024-11-14T09:45:01,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d8a71157c1b747699a0aea3d70e193cf to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d8a71157c1b747699a0aea3d70e193cf 2024-11-14T09:45:01,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/abb99721509a41f399aa67af26872526 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/abb99721509a41f399aa67af26872526 2024-11-14T09:45:01,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/efc34917e2da4dd5a79b14b0980fc17d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/efc34917e2da4dd5a79b14b0980fc17d 2024-11-14T09:45:01,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/c3c0f8b6397d473ab0e33027088dfba7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/c3c0f8b6397d473ab0e33027088dfba7 2024-11-14T09:45:01,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/890885b5ebc5443999d8aeedccc22011 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/890885b5ebc5443999d8aeedccc22011 2024-11-14T09:45:01,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/111e40ca252441b3ad933d2c36b1f0a3 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/111e40ca252441b3ad933d2c36b1f0a3 2024-11-14T09:45:01,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/423442709e594521b91f07e7e1e303ef to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/423442709e594521b91f07e7e1e303ef 2024-11-14T09:45:01,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2f64c5f839924e6f9b75e733293b92b0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2f64c5f839924e6f9b75e733293b92b0 2024-11-14T09:45:01,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d92e05a59dec4e3993e40234ffdddaa1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d92e05a59dec4e3993e40234ffdddaa1 2024-11-14T09:45:01,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d5658608e2604dc3a7f9510e7384da5d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/d5658608e2604dc3a7f9510e7384da5d 2024-11-14T09:45:01,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/de791e7b45a54649837b90cf6e5d5dec to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/de791e7b45a54649837b90cf6e5d5dec 2024-11-14T09:45:01,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/54e32252a946455fabd30e26354b4340 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/54e32252a946455fabd30e26354b4340 2024-11-14T09:45:01,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/9e5d07e25e3e41e3b0ecbde114486b34 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/9e5d07e25e3e41e3b0ecbde114486b34 2024-11-14T09:45:01,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/64605091a20e448cbf757f4b472edfa5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/64605091a20e448cbf757f4b472edfa5 2024-11-14T09:45:01,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bd608439a65a4f94b4f71ee2f9a2969d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/bd608439a65a4f94b4f71ee2f9a2969d 2024-11-14T09:45:01,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b097b8e9e79043e4a3245e7e6f6d9759 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b097b8e9e79043e4a3245e7e6f6d9759 2024-11-14T09:45:01,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/58c0bdc7f1e5405183a6fa36de19cc15 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/58c0bdc7f1e5405183a6fa36de19cc15 2024-11-14T09:45:01,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/895bda11c8b14dbc856282d35350a27d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/2fce20641cb84105a5c00186c5aedac2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4ffd520b24154da182e74921a9f2fbef, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ff5b2f318e254256a5e5bb683ec637ab, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/25ad8e9a128c4d3b822fd212c4d35fa8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/e781e3a0907a40e99960c9ea7eed9019, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/fa74787c732b492783d6b13d48eec3a5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/18ca91becc0849adb10431600ab61d6b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/c2b36e05d434422281e37c6e4ba13607, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ea6fdcae8a414dca80e0f464e2ea527e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/304a0cab025045218a361d13419229f2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/647068de1c19488e9464b92b1d015971, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f686e16f2e83459480076ff90b5ee780, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/65c13382dddd432d864f583d88209f9f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4668a00b929b486c8d1db2073c1bf7fd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/20b5ee7470df4e439d3e69022896e1d9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/d3d01b34edee4a5b8fe75b4a4d8df162, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f0afc671c512486aab8e981e2a78e69c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/b7d83a1d0a4b46e1a18aec9cad34a4b2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/95b5eea5a3b04e9389ab3f55f7d5569c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/98d74d7d0bc64e0d99b59ca85862eb86, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/31fa625d08a44392be93d93326db4c78, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4d3bf7d1845644e7b0a429b5903637f7] to archive 2024-11-14T09:45:01,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:45:01,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/895bda11c8b14dbc856282d35350a27d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/895bda11c8b14dbc856282d35350a27d 2024-11-14T09:45:01,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/2fce20641cb84105a5c00186c5aedac2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/2fce20641cb84105a5c00186c5aedac2 2024-11-14T09:45:01,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4ffd520b24154da182e74921a9f2fbef to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4ffd520b24154da182e74921a9f2fbef 2024-11-14T09:45:01,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ff5b2f318e254256a5e5bb683ec637ab to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ff5b2f318e254256a5e5bb683ec637ab 2024-11-14T09:45:01,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/25ad8e9a128c4d3b822fd212c4d35fa8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/25ad8e9a128c4d3b822fd212c4d35fa8 2024-11-14T09:45:01,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/e781e3a0907a40e99960c9ea7eed9019 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/e781e3a0907a40e99960c9ea7eed9019 2024-11-14T09:45:01,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/fa74787c732b492783d6b13d48eec3a5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/fa74787c732b492783d6b13d48eec3a5 2024-11-14T09:45:01,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/18ca91becc0849adb10431600ab61d6b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/18ca91becc0849adb10431600ab61d6b 2024-11-14T09:45:01,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/c2b36e05d434422281e37c6e4ba13607 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/c2b36e05d434422281e37c6e4ba13607 2024-11-14T09:45:01,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ea6fdcae8a414dca80e0f464e2ea527e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/ea6fdcae8a414dca80e0f464e2ea527e 2024-11-14T09:45:01,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/304a0cab025045218a361d13419229f2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/304a0cab025045218a361d13419229f2 2024-11-14T09:45:01,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/647068de1c19488e9464b92b1d015971 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/647068de1c19488e9464b92b1d015971 2024-11-14T09:45:01,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f686e16f2e83459480076ff90b5ee780 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f686e16f2e83459480076ff90b5ee780 2024-11-14T09:45:01,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/65c13382dddd432d864f583d88209f9f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/65c13382dddd432d864f583d88209f9f 2024-11-14T09:45:01,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4668a00b929b486c8d1db2073c1bf7fd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4668a00b929b486c8d1db2073c1bf7fd 2024-11-14T09:45:01,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/20b5ee7470df4e439d3e69022896e1d9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/20b5ee7470df4e439d3e69022896e1d9 2024-11-14T09:45:01,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/d3d01b34edee4a5b8fe75b4a4d8df162 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/d3d01b34edee4a5b8fe75b4a4d8df162 2024-11-14T09:45:01,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f0afc671c512486aab8e981e2a78e69c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/f0afc671c512486aab8e981e2a78e69c 2024-11-14T09:45:01,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/b7d83a1d0a4b46e1a18aec9cad34a4b2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/b7d83a1d0a4b46e1a18aec9cad34a4b2 2024-11-14T09:45:01,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/95b5eea5a3b04e9389ab3f55f7d5569c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/95b5eea5a3b04e9389ab3f55f7d5569c 2024-11-14T09:45:01,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/98d74d7d0bc64e0d99b59ca85862eb86 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/98d74d7d0bc64e0d99b59ca85862eb86 2024-11-14T09:45:01,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/31fa625d08a44392be93d93326db4c78 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/31fa625d08a44392be93d93326db4c78 2024-11-14T09:45:01,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4d3bf7d1845644e7b0a429b5903637f7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/4d3bf7d1845644e7b0a429b5903637f7 2024-11-14T09:45:01,575 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/recovered.edits/359.seqid, newMaxSeqId=359, maxSeqId=1 2024-11-14T09:45:01,576 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. 2024-11-14T09:45:01,576 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for ab93aec2a07aaf72eae598338b2d22a6: Waiting for close lock at 1731577500226Running coprocessor pre-close hooks at 1731577500226Disabling compacts and flushes for region at 1731577500226Disabling writes for close at 1731577500226Obtaining lock to block concurrent updates at 1731577500226Preparing flush snapshotting stores in ab93aec2a07aaf72eae598338b2d22a6 at 1731577500226Finished memstore snapshotting TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731577500227 (+1 ms)Flushing stores of TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6. at 1731577500228 (+1 ms)Flushing ab93aec2a07aaf72eae598338b2d22a6/A: creating writer at 1731577500228Flushing ab93aec2a07aaf72eae598338b2d22a6/A: appending metadata at 1731577500234 (+6 ms)Flushing ab93aec2a07aaf72eae598338b2d22a6/A: closing flushed file at 1731577500234Flushing ab93aec2a07aaf72eae598338b2d22a6/B: creating writer at 1731577500653 (+419 ms)Flushing ab93aec2a07aaf72eae598338b2d22a6/B: appending metadata at 1731577500656 (+3 ms)Flushing ab93aec2a07aaf72eae598338b2d22a6/B: closing flushed file at 1731577500656Flushing ab93aec2a07aaf72eae598338b2d22a6/C: creating writer at 1731577501070 (+414 ms)Flushing ab93aec2a07aaf72eae598338b2d22a6/C: appending metadata at 1731577501073 (+3 ms)Flushing ab93aec2a07aaf72eae598338b2d22a6/C: closing flushed file at 1731577501074 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ed155c4: reopening flushed file at 1731577501489 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@166ecf9c: reopening flushed file at 1731577501496 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43923785: reopening flushed file at 1731577501501 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ab93aec2a07aaf72eae598338b2d22a6 in 1281ms, sequenceid=356, compaction requested=true at 1731577501507 (+6 ms)Writing region close event to WAL at 1731577501573 (+66 ms)Running coprocessor post-close hooks at 1731577501576 (+3 ms)Closed at 1731577501576 2024-11-14T09:45:01,577 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:45:01,578 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=ab93aec2a07aaf72eae598338b2d22a6, regionState=CLOSED 2024-11-14T09:45:01,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:45:01,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T09:45:01,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure ab93aec2a07aaf72eae598338b2d22a6, server=defc576eb6b7,42511,1731577408298 in 1.5070 sec 2024-11-14T09:45:01,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-14T09:45:01,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ab93aec2a07aaf72eae598338b2d22a6, UNASSIGN in 1.5110 sec 2024-11-14T09:45:01,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T09:45:01,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-14T09:45:01,584 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577501584"}]},"ts":"1731577501584"} 2024-11-14T09:45:01,585 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:45:01,585 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:45:01,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5260 sec 2024-11-14T09:45:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:45:02,221 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:02,222 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:45:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:02,226 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:45:02,228 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:02,231 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:45:02,234 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/recovered.edits] 2024-11-14T09:45:02,287 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/a435878535a44ef7b90aa4989230ef26 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/a435878535a44ef7b90aa4989230ef26 2024-11-14T09:45:02,291 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d47a30c3840c4da38865bdbfc1ec9f4c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/d47a30c3840c4da38865bdbfc1ec9f4c 2024-11-14T09:45:02,293 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f2a7126a2180406fa79f88b89227793a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/A/f2a7126a2180406fa79f88b89227793a 2024-11-14T09:45:02,299 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2b91fad9a0e142aa8c3c2b65f8b2c7f2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/2b91fad9a0e142aa8c3c2b65f8b2c7f2 2024-11-14T09:45:02,302 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/3bb9a800d35f429aa3c1ccfda550b371 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/3bb9a800d35f429aa3c1ccfda550b371 2024-11-14T09:45:02,304 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b2857c14a7b0404aa2b753b8ff0b20b9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/B/b2857c14a7b0404aa2b753b8ff0b20b9 2024-11-14T09:45:02,310 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/0f31b50c2a6748d2b53539ecff7c0e85 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/0f31b50c2a6748d2b53539ecff7c0e85 2024-11-14T09:45:02,312 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/74838d54d0fa456a8040c31b9b53c09e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/74838d54d0fa456a8040c31b9b53c09e 2024-11-14T09:45:02,314 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/a7ef213100474de4879d64401581511d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/C/a7ef213100474de4879d64401581511d 2024-11-14T09:45:02,319 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/recovered.edits/359.seqid to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6/recovered.edits/359.seqid 2024-11-14T09:45:02,320 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ab93aec2a07aaf72eae598338b2d22a6 2024-11-14T09:45:02,320 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:45:02,324 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:45:02,331 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:45:02,333 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:45:02,334 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:02,334 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:45:02,335 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577502334"}]},"ts":"9223372036854775807"} 2024-11-14T09:45:02,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:45:02,337 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ab93aec2a07aaf72eae598338b2d22a6, NAME => 'TestAcidGuarantees,,1731577470726.ab93aec2a07aaf72eae598338b2d22a6.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:45:02,337 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:45:02,337 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577502337"}]},"ts":"9223372036854775807"} 2024-11-14T09:45:02,339 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:45:02,339 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:02,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 117 msec 2024-11-14T09:45:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:45:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:45:02,546 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:02,551 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 251), OpenFileDescriptor=455 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=313 (was 369), ProcessCount=11 (was 11), AvailableMemoryMB=3837 (was 3905) 2024-11-14T09:45:02,559 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=313, ProcessCount=11, AvailableMemoryMB=3837 2024-11-14T09:45:02,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:45:02,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:45:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:02,562 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:45:02,562 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:02,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-14T09:45:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T09:45:02,563 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:45:02,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742131_1307 (size=1061) 2024-11-14T09:45:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T09:45:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T09:45:02,973 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:45:02,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742132_1308 (size=53) 2024-11-14T09:45:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T09:45:03,384 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:45:03,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 29dbb160edffb51f36ebfbb295b932e8, disabling compactions & flushes 2024-11-14T09:45:03,385 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. after waiting 0 ms 2024-11-14T09:45:03,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,385 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29dbb160edffb51f36ebfbb295b932e8: Waiting for close lock at 1731577503385Disabling compacts and flushes for region at 1731577503385Disabling writes for close at 1731577503385Writing region close event to WAL at 1731577503385Closed at 1731577503385 2024-11-14T09:45:03,388 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:45:03,389 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577503388"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577503388"}]},"ts":"1731577503388"} 2024-11-14T09:45:03,392 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:45:03,393 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:45:03,393 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577503393"}]},"ts":"1731577503393"} 2024-11-14T09:45:03,395 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:45:03,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, ASSIGN}] 2024-11-14T09:45:03,396 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, ASSIGN 2024-11-14T09:45:03,397 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, ASSIGN; state=OFFLINE, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=false 2024-11-14T09:45:03,548 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:03,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, ASSIGN because future has completed 2024-11-14T09:45:03,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:45:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T09:45:03,710 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,710 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:45:03,710 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,710 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:45:03,711 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,711 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,713 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,715 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:03,715 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29dbb160edffb51f36ebfbb295b932e8 columnFamilyName A 2024-11-14T09:45:03,716 DEBUG [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:03,716 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(327): Store=29dbb160edffb51f36ebfbb295b932e8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:03,717 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,718 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:03,719 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29dbb160edffb51f36ebfbb295b932e8 columnFamilyName B 2024-11-14T09:45:03,719 DEBUG [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:03,719 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(327): Store=29dbb160edffb51f36ebfbb295b932e8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:03,720 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,721 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:03,721 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29dbb160edffb51f36ebfbb295b932e8 columnFamilyName C 2024-11-14T09:45:03,721 DEBUG [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:03,721 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(327): Store=29dbb160edffb51f36ebfbb295b932e8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:03,721 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,721 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,722 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,722 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,723 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,723 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,723 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:45:03,724 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,726 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:45:03,726 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 29dbb160edffb51f36ebfbb295b932e8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60280112, jitterRate=-0.10175633430480957}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:45:03,726 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:03,727 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 29dbb160edffb51f36ebfbb295b932e8: Running coprocessor pre-open hook at 1731577503711Writing region info on filesystem at 1731577503711Initializing all the Stores at 1731577503712 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577503712Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577503713 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577503713Cleaning up temporary data from old regions at 1731577503723 (+10 ms)Running coprocessor post-open hooks at 1731577503726 (+3 ms)Region opened successfully at 1731577503727 (+1 ms) 2024-11-14T09:45:03,728 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., pid=81, masterSystemTime=1731577503705 2024-11-14T09:45:03,730 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,730 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:03,730 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:03,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:45:03,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T09:45:03,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 in 181 msec 2024-11-14T09:45:03,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-14T09:45:03,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, ASSIGN in 340 msec 2024-11-14T09:45:03,737 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:45:03,737 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577503737"}]},"ts":"1731577503737"} 2024-11-14T09:45:03,738 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:45:03,739 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:45:03,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-14T09:45:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-14T09:45:04,710 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:04,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:04,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:04,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:04,713 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f71598d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:04,715 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56602bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:04,716 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:04,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:04,719 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:45:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,33347,1731577407591 2024-11-14T09:45:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e0e8299 2024-11-14T09:45:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:45:04,721 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:45:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:45:04,723 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:45:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:04,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742133_1309 (size=1097) 2024-11-14T09:45:05,142 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T09:45:05,142 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T09:45:05,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:45:05,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, REOPEN/MOVE}] 2024-11-14T09:45:05,150 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, REOPEN/MOVE 2024-11-14T09:45:05,151 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, REOPEN/MOVE because future has completed 2024-11-14T09:45:05,153 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:45:05,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:45:05,305 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,305 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:45:05,305 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 29dbb160edffb51f36ebfbb295b932e8, disabling compactions & flushes 2024-11-14T09:45:05,305 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,305 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,305 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. after waiting 0 ms 2024-11-14T09:45:05,305 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,309 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:45:05,310 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,310 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 29dbb160edffb51f36ebfbb295b932e8: Waiting for close lock at 1731577505305Running coprocessor pre-close hooks at 1731577505305Disabling compacts and flushes for region at 1731577505305Disabling writes for close at 1731577505305Writing region close event to WAL at 1731577505306 (+1 ms)Running coprocessor post-close hooks at 1731577505310 (+4 ms)Closed at 1731577505310 2024-11-14T09:45:05,310 WARN [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 29dbb160edffb51f36ebfbb295b932e8 to self. 2024-11-14T09:45:05,312 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,312 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=CLOSED 2024-11-14T09:45:05,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:45:05,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-14T09:45:05,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 in 161 msec 2024-11-14T09:45:05,317 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, REOPEN/MOVE; state=CLOSED, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=true 2024-11-14T09:45:05,467 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, REOPEN/MOVE because future has completed 2024-11-14T09:45:05,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:45:05,630 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,630 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:45:05,630 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,631 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:45:05,631 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,631 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,632 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,633 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:05,634 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29dbb160edffb51f36ebfbb295b932e8 columnFamilyName A 2024-11-14T09:45:05,635 DEBUG [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:05,635 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(327): Store=29dbb160edffb51f36ebfbb295b932e8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:05,636 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,637 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:05,637 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29dbb160edffb51f36ebfbb295b932e8 columnFamilyName B 2024-11-14T09:45:05,637 DEBUG [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:05,637 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(327): Store=29dbb160edffb51f36ebfbb295b932e8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:05,638 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,638 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:05,638 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29dbb160edffb51f36ebfbb295b932e8 columnFamilyName C 2024-11-14T09:45:05,638 DEBUG [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:05,639 INFO [StoreOpener-29dbb160edffb51f36ebfbb295b932e8-1 {}] regionserver.HStore(327): Store=29dbb160edffb51f36ebfbb295b932e8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:05,639 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,639 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,640 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,641 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,643 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,643 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,644 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:45:05,645 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,646 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 29dbb160edffb51f36ebfbb295b932e8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61451421, jitterRate=-0.08430247008800507}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:45:05,646 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,646 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 29dbb160edffb51f36ebfbb295b932e8: Running coprocessor pre-open hook at 1731577505631Writing region info on filesystem at 1731577505631Initializing all the Stores at 1731577505632 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577505632Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577505632Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577505632Cleaning up temporary data from old regions at 1731577505643 (+11 ms)Running coprocessor post-open hooks at 1731577505646 (+3 ms)Region opened successfully at 1731577505646 2024-11-14T09:45:05,647 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., pid=86, masterSystemTime=1731577505621 2024-11-14T09:45:05,649 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,649 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=OPEN, openSeqNum=5, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:45:05,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-14T09:45:05,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 in 182 msec 2024-11-14T09:45:05,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-14T09:45:05,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, REOPEN/MOVE in 503 msec 2024-11-14T09:45:05,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T09:45:05,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-14T09:45:05,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-11-14T09:45:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T09:45:05,659 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:05,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c43e443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,662 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e2367e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,664 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1998941e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d99fd5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50af6015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7046d280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2e36cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1710356a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c16eaac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258d589e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a196e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35bd4d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,680 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@360b1449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,681 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbb1d1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711321b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e7c2d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a8be3df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eda16ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:05,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:05,691 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327259be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:05,692 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:05,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756b7b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:05,694 DEBUG [Thread-1441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,694 DEBUG [Thread-1430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,694 DEBUG [Thread-1432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,694 DEBUG [Thread-1443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:05,704 DEBUG [Thread-1435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-14T09:45:05,706 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:05,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:45:05,706 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:05,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:05,707 DEBUG [Thread-1437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,708 DEBUG [Thread-1437 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,708 DEBUG [Thread-1437 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,709 DEBUG [Thread-1428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,710 DEBUG [Thread-1424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,710 DEBUG [Thread-1426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,712 DEBUG [Thread-1439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:05,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:05,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:45:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:05,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:45:05,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114702f446e96cd402180a6ae05bc6eda52_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577505708/Put/seqid=0 2024-11-14T09:45:05,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742134_1310 (size=14594) 2024-11-14T09:45:05,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577505745, max timestamp=1731577505758Num uniques 859; 2024-11-14T09:45:05,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:05,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577505745, max timestamp=1731577505758Num uniques 859; 2024-11-14T09:45:05,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577505745, max timestamp=1731577505758Num uniques 859; 2024-11-14T09:45:05,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:05,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:05,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577565780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577565780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577565789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577565790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577565791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:45:05,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:05,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577565900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577565900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577565900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577565910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577565910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:05,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:45:06,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577566110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577566120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577566120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577566130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577566131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:06,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114702f446e96cd402180a6ae05bc6eda52_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114702f446e96cd402180a6ae05bc6eda52_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:06,162 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/b4d7a126b72f40f09d468d927590449b, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:06,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/b4d7a126b72f40f09d468d927590449b is 175, key is test_row_0/A:col10/1731577505708/Put/seqid=0 2024-11-14T09:45:06,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742135_1311 (size=39545) 2024-11-14T09:45:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:45:06,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577566420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577566430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577566430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577566440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577566440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/b4d7a126b72f40f09d468d927590449b 2024-11-14T09:45:06,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/867181ccbcca416ca0f7668a36651960 is 50, key is test_row_0/B:col10/1731577505708/Put/seqid=0 2024-11-14T09:45:06,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742136_1312 (size=9657) 2024-11-14T09:45:06,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:45:06,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:06,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:06,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577566940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577566940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577566950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577566950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577566950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:07,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/867181ccbcca416ca0f7668a36651960 2024-11-14T09:45:07,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/35e45773cfee4a51ac1462d05306e477 is 50, key is test_row_0/C:col10/1731577505708/Put/seqid=0 2024-11-14T09:45:07,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742137_1313 (size=9657) 2024-11-14T09:45:07,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:07,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,103 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:45:07,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:07,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:07,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:07,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:07,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:07,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:07,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/35e45773cfee4a51ac1462d05306e477 2024-11-14T09:45:07,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/b4d7a126b72f40f09d468d927590449b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/b4d7a126b72f40f09d468d927590449b 2024-11-14T09:45:07,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/b4d7a126b72f40f09d468d927590449b, entries=200, sequenceid=14, filesize=38.6 K 2024-11-14T09:45:07,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/867181ccbcca416ca0f7668a36651960 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/867181ccbcca416ca0f7668a36651960 2024-11-14T09:45:07,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/867181ccbcca416ca0f7668a36651960, entries=100, sequenceid=14, filesize=9.4 K 2024-11-14T09:45:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/35e45773cfee4a51ac1462d05306e477 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/35e45773cfee4a51ac1462d05306e477 2024-11-14T09:45:07,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/35e45773cfee4a51ac1462d05306e477, entries=100, sequenceid=14, filesize=9.4 K 2024-11-14T09:45:07,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 29dbb160edffb51f36ebfbb295b932e8 in 1731ms, sequenceid=14, compaction requested=false 2024-11-14T09:45:07,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:07,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-14T09:45:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:07,540 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T09:45:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:07,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ba140c790b7d4ef49bf5c2aa46fab879_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577505790/Put/seqid=0 2024-11-14T09:45:07,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742138_1314 (size=12154) 2024-11-14T09:45:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:45:07,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:07,959 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ba140c790b7d4ef49bf5c2aa46fab879_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ba140c790b7d4ef49bf5c2aa46fab879_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/9ba9852f9dda43aead9854540519b9af, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/9ba9852f9dda43aead9854540519b9af is 175, key is test_row_0/A:col10/1731577505790/Put/seqid=0 2024-11-14T09:45:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:07,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742139_1315 (size=30955) 2024-11-14T09:45:07,972 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/9ba9852f9dda43aead9854540519b9af 2024-11-14T09:45:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/8ff4fa692887412fbf1e77da28028cf6 is 50, key is test_row_0/B:col10/1731577505790/Put/seqid=0 2024-11-14T09:45:07,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577567976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:07,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742140_1316 (size=12001) 2024-11-14T09:45:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:07,989 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/8ff4fa692887412fbf1e77da28028cf6 2024-11-14T09:45:07,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/2cf6f15c81dd42da9661e0697dfbf1dc is 50, key is test_row_0/C:col10/1731577505790/Put/seqid=0 2024-11-14T09:45:07,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577567983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:07,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577567984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577567986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577567987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742141_1317 (size=12001) 2024-11-14T09:45:08,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577568090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577568100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577568101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577568101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577568110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577568310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577568320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577568320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577568320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577568320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,402 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/2cf6f15c81dd42da9661e0697dfbf1dc 2024-11-14T09:45:08,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/9ba9852f9dda43aead9854540519b9af as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/9ba9852f9dda43aead9854540519b9af 2024-11-14T09:45:08,409 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/9ba9852f9dda43aead9854540519b9af, entries=150, sequenceid=49, filesize=30.2 K 2024-11-14T09:45:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/8ff4fa692887412fbf1e77da28028cf6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/8ff4fa692887412fbf1e77da28028cf6 2024-11-14T09:45:08,414 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/8ff4fa692887412fbf1e77da28028cf6, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:45:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/2cf6f15c81dd42da9661e0697dfbf1dc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/2cf6f15c81dd42da9661e0697dfbf1dc 2024-11-14T09:45:08,418 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/2cf6f15c81dd42da9661e0697dfbf1dc, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T09:45:08,418 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 29dbb160edffb51f36ebfbb295b932e8 in 878ms, sequenceid=49, compaction requested=false 2024-11-14T09:45:08,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-14T09:45:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-14T09:45:08,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-14T09:45:08,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7130 sec 2024-11-14T09:45:08,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 2.7180 sec 2024-11-14T09:45:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:08,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:45:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:08,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146d0e976a30ba4cc8a7b92fd72ba18ebf_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577507985/Put/seqid=0 2024-11-14T09:45:08,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742142_1318 (size=14594) 2024-11-14T09:45:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577568690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577568690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577508652, max timestamp=1731577508706Num uniques 859; 2024-11-14T09:45:08,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577508652, max timestamp=1731577508706Num uniques 859; 2024-11-14T09:45:08,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:08,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:08,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577508652, max timestamp=1731577508706Num uniques 859; 2024-11-14T09:45:08,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577568747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577568748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577568754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577568810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577568811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577568860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577568870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577568870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:08,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577569031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577569031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,041 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:09,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146d0e976a30ba4cc8a7b92fd72ba18ebf_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d0e976a30ba4cc8a7b92fd72ba18ebf_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:09,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6330d6583ea2427aa8035f451fd4b19d, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:09,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6330d6583ea2427aa8035f451fd4b19d is 175, key is test_row_0/A:col10/1731577507985/Put/seqid=0 2024-11-14T09:45:09,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742143_1319 (size=39549) 2024-11-14T09:45:09,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577569080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577569080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577569080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577569350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577569351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577569400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577569400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577569400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6330d6583ea2427aa8035f451fd4b19d 2024-11-14T09:45:09,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/c35e19ccef754a6d9c6ab6809bdf11f2 is 50, key is test_row_0/B:col10/1731577507985/Put/seqid=0 2024-11-14T09:45:09,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742144_1320 (size=12001) 2024-11-14T09:45:09,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/c35e19ccef754a6d9c6ab6809bdf11f2 2024-11-14T09:45:09,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/309f9e8b6e4c41589bc04a9b239caf37 is 50, key is test_row_0/C:col10/1731577507985/Put/seqid=0 2024-11-14T09:45:09,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742145_1321 (size=12001) 2024-11-14T09:45:09,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/309f9e8b6e4c41589bc04a9b239caf37 2024-11-14T09:45:09,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577569860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-14T09:45:09,870 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:09,871 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-14T09:45:09,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6330d6583ea2427aa8035f451fd4b19d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6330d6583ea2427aa8035f451fd4b19d 2024-11-14T09:45:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T09:45:09,872 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:09,873 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:09,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:09,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6330d6583ea2427aa8035f451fd4b19d, entries=200, sequenceid=60, filesize=38.6 K 2024-11-14T09:45:09,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/c35e19ccef754a6d9c6ab6809bdf11f2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/c35e19ccef754a6d9c6ab6809bdf11f2 2024-11-14T09:45:09,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577569870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/c35e19ccef754a6d9c6ab6809bdf11f2, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:45:09,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/309f9e8b6e4c41589bc04a9b239caf37 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/309f9e8b6e4c41589bc04a9b239caf37 2024-11-14T09:45:09,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/309f9e8b6e4c41589bc04a9b239caf37, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:45:09,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 29dbb160edffb51f36ebfbb295b932e8 in 1256ms, sequenceid=60, compaction requested=true 2024-11-14T09:45:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:09,882 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:09,882 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:09,883 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:09,883 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:09,883 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/A is initiating minor compaction (all files) 2024-11-14T09:45:09,883 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/B is initiating minor compaction (all files) 2024-11-14T09:45:09,883 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/A in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:09,883 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/B in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:09,883 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/b4d7a126b72f40f09d468d927590449b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/9ba9852f9dda43aead9854540519b9af, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6330d6583ea2427aa8035f451fd4b19d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=107.5 K 2024-11-14T09:45:09,883 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/867181ccbcca416ca0f7668a36651960, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/8ff4fa692887412fbf1e77da28028cf6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/c35e19ccef754a6d9c6ab6809bdf11f2] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=32.9 K 2024-11-14T09:45:09,883 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:09,883 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/b4d7a126b72f40f09d468d927590449b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/9ba9852f9dda43aead9854540519b9af, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6330d6583ea2427aa8035f451fd4b19d] 2024-11-14T09:45:09,884 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 867181ccbcca416ca0f7668a36651960, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577505705 2024-11-14T09:45:09,884 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4d7a126b72f40f09d468d927590449b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577505702 2024-11-14T09:45:09,884 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ba9852f9dda43aead9854540519b9af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731577505788 2024-11-14T09:45:09,884 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ff4fa692887412fbf1e77da28028cf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731577505788 2024-11-14T09:45:09,884 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6330d6583ea2427aa8035f451fd4b19d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577507983 2024-11-14T09:45:09,884 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c35e19ccef754a6d9c6ab6809bdf11f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577507984 2024-11-14T09:45:09,892 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:09,894 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148ce0c8dc8c4645bea2c871ae329e12cc_29dbb160edffb51f36ebfbb295b932e8 store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:09,894 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:09,894 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/b09e4aae92fb4b6ebe41a2b0aa5b32b8 is 50, key is test_row_0/B:col10/1731577507985/Put/seqid=0 2024-11-14T09:45:09,896 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148ce0c8dc8c4645bea2c871ae329e12cc_29dbb160edffb51f36ebfbb295b932e8, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:09,896 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148ce0c8dc8c4645bea2c871ae329e12cc_29dbb160edffb51f36ebfbb295b932e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:09,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742146_1322 (size=12104) 2024-11-14T09:45:09,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742147_1323 (size=4469) 2024-11-14T09:45:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:09,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T09:45:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:09,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f7f110cd69724543b9f4798dc79355ce_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577508747/Put/seqid=0 2024-11-14T09:45:09,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742148_1324 (size=14594) 2024-11-14T09:45:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577569946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577569947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577569948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T09:45:10,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577570060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577570060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577570070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:10,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T09:45:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577570280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577570280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577570280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,303 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/b09e4aae92fb4b6ebe41a2b0aa5b32b8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/b09e4aae92fb4b6ebe41a2b0aa5b32b8 2024-11-14T09:45:10,306 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#A#compaction#273 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:10,307 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5f6882b306c046caa8cb1e3de1843329 is 175, key is test_row_0/A:col10/1731577507985/Put/seqid=0 2024-11-14T09:45:10,308 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/B of 29dbb160edffb51f36ebfbb295b932e8 into b09e4aae92fb4b6ebe41a2b0aa5b32b8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:10,308 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:10,308 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/B, priority=13, startTime=1731577509882; duration=0sec 2024-11-14T09:45:10,308 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:10,308 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:B 2024-11-14T09:45:10,308 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:10,309 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:10,309 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/C is initiating minor compaction (all files) 2024-11-14T09:45:10,309 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/C in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,309 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/35e45773cfee4a51ac1462d05306e477, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/2cf6f15c81dd42da9661e0697dfbf1dc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/309f9e8b6e4c41589bc04a9b239caf37] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=32.9 K 2024-11-14T09:45:10,309 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e45773cfee4a51ac1462d05306e477, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577505705 2024-11-14T09:45:10,309 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf6f15c81dd42da9661e0697dfbf1dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731577505788 2024-11-14T09:45:10,310 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 309f9e8b6e4c41589bc04a9b239caf37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577507984 2024-11-14T09:45:10,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742149_1325 (size=31058) 2024-11-14T09:45:10,316 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5f6882b306c046caa8cb1e3de1843329 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5f6882b306c046caa8cb1e3de1843329 2024-11-14T09:45:10,318 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:10,318 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/185a1d7054314e4c9d8bc681968defb7 is 50, key is test_row_0/C:col10/1731577507985/Put/seqid=0 2024-11-14T09:45:10,320 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/A of 29dbb160edffb51f36ebfbb295b932e8 into 5f6882b306c046caa8cb1e3de1843329(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:10,320 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:10,320 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/A, priority=13, startTime=1731577509882; duration=0sec 2024-11-14T09:45:10,320 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:10,320 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:A 2024-11-14T09:45:10,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742150_1326 (size=12104) 2024-11-14T09:45:10,325 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/185a1d7054314e4c9d8bc681968defb7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/185a1d7054314e4c9d8bc681968defb7 2024-11-14T09:45:10,329 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/C of 29dbb160edffb51f36ebfbb295b932e8 into 185a1d7054314e4c9d8bc681968defb7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:10,329 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:10,329 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/C, priority=13, startTime=1731577509882; duration=0sec 2024-11-14T09:45:10,329 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:10,329 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:C 2024-11-14T09:45:10,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,339 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:10,343 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f7f110cd69724543b9f4798dc79355ce_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f7f110cd69724543b9f4798dc79355ce_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:10,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/31ce95e891c64b159bead14fac4e835f, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:10,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/31ce95e891c64b159bead14fac4e835f is 175, key is test_row_0/A:col10/1731577508747/Put/seqid=0 2024-11-14T09:45:10,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742151_1327 (size=39549) 2024-11-14T09:45:10,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:10,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:10,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T09:45:10,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577570590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577570600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577570600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:10,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:10,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,751 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/31ce95e891c64b159bead14fac4e835f 2024-11-14T09:45:10,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/1c70d9b3520745c1ab02f773c231dac7 is 50, key is test_row_0/B:col10/1731577508747/Put/seqid=0 2024-11-14T09:45:10,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742152_1328 (size=12001) 2024-11-14T09:45:10,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:10,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577570880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,884 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 29dbb160edffb51f36ebfbb295b932e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577570890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:10,894 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 29dbb160edffb51f36ebfbb295b932e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:10,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:10,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T09:45:11,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577571100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:11,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577571110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:11,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:11,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:11,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:11,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577571120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:11,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/1c70d9b3520745c1ab02f773c231dac7 2024-11-14T09:45:11,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/a06fede6b96d43f5a97ed10d32510c3c is 50, key is test_row_0/C:col10/1731577508747/Put/seqid=0 2024-11-14T09:45:11,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742153_1329 (size=12001) 2024-11-14T09:45:11,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:11,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:11,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:11,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:11,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:11,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:11,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:11,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/a06fede6b96d43f5a97ed10d32510c3c 2024-11-14T09:45:11,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/31ce95e891c64b159bead14fac4e835f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/31ce95e891c64b159bead14fac4e835f 2024-11-14T09:45:11,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/31ce95e891c64b159bead14fac4e835f, entries=200, sequenceid=95, filesize=38.6 K 2024-11-14T09:45:11,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/1c70d9b3520745c1ab02f773c231dac7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1c70d9b3520745c1ab02f773c231dac7 2024-11-14T09:45:11,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1c70d9b3520745c1ab02f773c231dac7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T09:45:11,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/a06fede6b96d43f5a97ed10d32510c3c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/a06fede6b96d43f5a97ed10d32510c3c 2024-11-14T09:45:11,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/a06fede6b96d43f5a97ed10d32510c3c, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T09:45:11,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 29dbb160edffb51f36ebfbb295b932e8 in 1665ms, sequenceid=95, compaction requested=false 2024-11-14T09:45:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:11,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T09:45:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:11,704 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:45:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111442cd9bf286b44bb39079dc277f891326_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577509946/Put/seqid=0 2024-11-14T09:45:11,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742154_1330 (size=12154) 2024-11-14T09:45:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T09:45:12,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:12,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:12,125 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111442cd9bf286b44bb39079dc277f891326_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111442cd9bf286b44bb39079dc277f891326_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:12,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:12,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1 is 175, key is test_row_0/A:col10/1731577509946/Put/seqid=0 2024-11-14T09:45:12,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742155_1331 (size=30955) 2024-11-14T09:45:12,133 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1 2024-11-14T09:45:12,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/35fa1d366c854b179b6616faaeacee29 is 50, key is test_row_0/B:col10/1731577509946/Put/seqid=0 2024-11-14T09:45:12,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742156_1332 (size=12001) 2024-11-14T09:45:12,154 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/35fa1d366c854b179b6616faaeacee29 2024-11-14T09:45:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/98fdf6c582944d698f7357f460cc7582 is 50, key is test_row_0/C:col10/1731577509946/Put/seqid=0 2024-11-14T09:45:12,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742157_1333 (size=12001) 2024-11-14T09:45:12,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577512180, max timestamp=1731577512192Num uniques 859; 2024-11-14T09:45:12,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577512180, max timestamp=1731577512192Num uniques 859; 2024-11-14T09:45:12,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577512180, max timestamp=1731577512192Num uniques 859; 2024-11-14T09:45:12,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:12,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:12,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:12,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577572294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577572311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577572312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577572420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577572420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577572430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,569 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/98fdf6c582944d698f7357f460cc7582 2024-11-14T09:45:12,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1 2024-11-14T09:45:12,576 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1, entries=150, sequenceid=108, filesize=30.2 K 2024-11-14T09:45:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/35fa1d366c854b179b6616faaeacee29 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/35fa1d366c854b179b6616faaeacee29 2024-11-14T09:45:12,580 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/35fa1d366c854b179b6616faaeacee29, entries=150, sequenceid=108, filesize=11.7 K 2024-11-14T09:45:12,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/98fdf6c582944d698f7357f460cc7582 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/98fdf6c582944d698f7357f460cc7582 2024-11-14T09:45:12,584 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/98fdf6c582944d698f7357f460cc7582, entries=150, sequenceid=108, filesize=11.7 K 2024-11-14T09:45:12,585 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 29dbb160edffb51f36ebfbb295b932e8 in 881ms, sequenceid=108, compaction requested=true 2024-11-14T09:45:12,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:12,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:12,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-14T09:45:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-14T09:45:12,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T09:45:12,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7140 sec 2024-11-14T09:45:12,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.7180 sec 2024-11-14T09:45:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:12,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-14T09:45:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:45:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:45:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:45:12,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142d70919d630b4c78b51e72b63983ac24_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577512311/Put/seqid=0 2024-11-14T09:45:12,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742158_1334 (size=14794) 2024-11-14T09:45:12,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577572664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577572668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577572669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577572780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577572780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577572790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577572900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,908 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 29dbb160edffb51f36ebfbb295b932e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:12,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577572910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:12,922 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 29dbb160edffb51f36ebfbb295b932e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577573000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577573000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577573000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:13,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142d70919d630b4c78b51e72b63983ac24_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142d70919d630b4c78b51e72b63983ac24_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:13,063 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/ad9bc55e8cd046a483a902d6dc0bc004, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:13,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/ad9bc55e8cd046a483a902d6dc0bc004 is 175, key is test_row_0/A:col10/1731577512311/Put/seqid=0 2024-11-14T09:45:13,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742159_1335 (size=39749) 2024-11-14T09:45:13,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577573321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577573321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577573321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/ad9bc55e8cd046a483a902d6dc0bc004 2024-11-14T09:45:13,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/0c8ac46825e6485491e198cc1d80c13f is 50, key is test_row_0/B:col10/1731577512311/Put/seqid=0 2024-11-14T09:45:13,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742160_1336 (size=12151) 2024-11-14T09:45:13,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577573840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577573840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577573840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:13,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/0c8ac46825e6485491e198cc1d80c13f 2024-11-14T09:45:13,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/34b2fea56a7d46d8b57e22dfd9a90fb6 is 50, key is test_row_0/C:col10/1731577512311/Put/seqid=0 2024-11-14T09:45:13,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742161_1337 (size=12151) 2024-11-14T09:45:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T09:45:14,030 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:14,031 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-14T09:45:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T09:45:14,032 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:14,033 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:14,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T09:45:14,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T09:45:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:14,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:14,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/34b2fea56a7d46d8b57e22dfd9a90fb6 2024-11-14T09:45:14,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/ad9bc55e8cd046a483a902d6dc0bc004 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/ad9bc55e8cd046a483a902d6dc0bc004 2024-11-14T09:45:14,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/ad9bc55e8cd046a483a902d6dc0bc004, entries=200, sequenceid=146, filesize=38.8 K 2024-11-14T09:45:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/0c8ac46825e6485491e198cc1d80c13f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/0c8ac46825e6485491e198cc1d80c13f 2024-11-14T09:45:14,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/0c8ac46825e6485491e198cc1d80c13f, entries=150, sequenceid=146, filesize=11.9 K 2024-11-14T09:45:14,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/34b2fea56a7d46d8b57e22dfd9a90fb6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/34b2fea56a7d46d8b57e22dfd9a90fb6 2024-11-14T09:45:14,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/34b2fea56a7d46d8b57e22dfd9a90fb6, entries=150, sequenceid=146, filesize=11.9 K 2024-11-14T09:45:14,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 29dbb160edffb51f36ebfbb295b932e8 in 1657ms, sequenceid=146, compaction requested=true 2024-11-14T09:45:14,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:14,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:14,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:14,304 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:14,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:14,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:14,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:14,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:14,305 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/A is initiating minor compaction (all files) 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/B is initiating minor compaction (all files) 2024-11-14T09:45:14,306 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/B in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:14,306 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/A in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:14,306 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5f6882b306c046caa8cb1e3de1843329, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/31ce95e891c64b159bead14fac4e835f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/ad9bc55e8cd046a483a902d6dc0bc004] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=138.0 K 2024-11-14T09:45:14,306 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/b09e4aae92fb4b6ebe41a2b0aa5b32b8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1c70d9b3520745c1ab02f773c231dac7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/35fa1d366c854b179b6616faaeacee29, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/0c8ac46825e6485491e198cc1d80c13f] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=47.1 K 2024-11-14T09:45:14,306 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5f6882b306c046caa8cb1e3de1843329, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/31ce95e891c64b159bead14fac4e835f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/ad9bc55e8cd046a483a902d6dc0bc004] 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b09e4aae92fb4b6ebe41a2b0aa5b32b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577507984 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6882b306c046caa8cb1e3de1843329, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577507984 2024-11-14T09:45:14,306 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c70d9b3520745c1ab02f773c231dac7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731577508744 2024-11-14T09:45:14,307 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ce95e891c64b159bead14fac4e835f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731577508744 2024-11-14T09:45:14,307 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27fe3d1f7ef7447ca0f3f9ba0eeaa0f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731577509929 2024-11-14T09:45:14,307 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 35fa1d366c854b179b6616faaeacee29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731577509929 2024-11-14T09:45:14,307 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9bc55e8cd046a483a902d6dc0bc004, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731577512228 2024-11-14T09:45:14,307 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8ac46825e6485491e198cc1d80c13f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731577512228 2024-11-14T09:45:14,312 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:14,315 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111466d3b4d2afe74dc19aeaae627270500b_29dbb160edffb51f36ebfbb295b932e8 store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:14,323 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:14,323 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/f415d7d0aba94eb2bbe6b4de5eec216e is 50, key is test_row_0/B:col10/1731577512311/Put/seqid=0 2024-11-14T09:45:14,327 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111466d3b4d2afe74dc19aeaae627270500b_29dbb160edffb51f36ebfbb295b932e8, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:14,328 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111466d3b4d2afe74dc19aeaae627270500b_29dbb160edffb51f36ebfbb295b932e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:14,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T09:45:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:14,339 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T09:45:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:14,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742162_1338 (size=12391) 2024-11-14T09:45:14,346 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/f415d7d0aba94eb2bbe6b4de5eec216e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f415d7d0aba94eb2bbe6b4de5eec216e 2024-11-14T09:45:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T09:45:14,351 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/B of 29dbb160edffb51f36ebfbb295b932e8 into f415d7d0aba94eb2bbe6b4de5eec216e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:14,351 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:14,351 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/B, priority=12, startTime=1731577514304; duration=0sec 2024-11-14T09:45:14,351 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:14,351 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:B 2024-11-14T09:45:14,351 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:14,353 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:14,353 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/C is initiating minor compaction (all files) 2024-11-14T09:45:14,353 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/C in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:14,353 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/185a1d7054314e4c9d8bc681968defb7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/a06fede6b96d43f5a97ed10d32510c3c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/98fdf6c582944d698f7357f460cc7582, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/34b2fea56a7d46d8b57e22dfd9a90fb6] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=47.1 K 2024-11-14T09:45:14,353 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 185a1d7054314e4c9d8bc681968defb7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577507984 2024-11-14T09:45:14,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742163_1339 (size=4469) 2024-11-14T09:45:14,354 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a06fede6b96d43f5a97ed10d32510c3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731577508744 2024-11-14T09:45:14,354 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 98fdf6c582944d698f7357f460cc7582, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731577509929 2024-11-14T09:45:14,354 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#A#compaction#285 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:14,355 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 34b2fea56a7d46d8b57e22dfd9a90fb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731577512228 2024-11-14T09:45:14,355 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/d4ca741ee78048b0be2959da3846d1ad is 175, key is test_row_0/A:col10/1731577512311/Put/seqid=0 2024-11-14T09:45:14,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142ff4cec269464c79943919baf7ef66cb_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577512663/Put/seqid=0 2024-11-14T09:45:14,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742164_1340 (size=31345) 2024-11-14T09:45:14,363 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:14,363 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/79bff444c4eb480a86111b2bcf54f34d is 50, key is test_row_0/C:col10/1731577512311/Put/seqid=0 2024-11-14T09:45:14,365 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/d4ca741ee78048b0be2959da3846d1ad as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/d4ca741ee78048b0be2959da3846d1ad 2024-11-14T09:45:14,370 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/A of 29dbb160edffb51f36ebfbb295b932e8 into d4ca741ee78048b0be2959da3846d1ad(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:14,370 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:14,370 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/A, priority=12, startTime=1731577514304; duration=0sec 2024-11-14T09:45:14,370 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:14,370 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:A 2024-11-14T09:45:14,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742165_1341 (size=12304) 2024-11-14T09:45:14,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:14,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742166_1342 (size=12391) 2024-11-14T09:45:14,375 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142ff4cec269464c79943919baf7ef66cb_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142ff4cec269464c79943919baf7ef66cb_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/60ca7a19cc77409da7339cad35ef0781, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/60ca7a19cc77409da7339cad35ef0781 is 175, key is test_row_0/A:col10/1731577512663/Put/seqid=0 2024-11-14T09:45:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742167_1343 (size=31105) 2024-11-14T09:45:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T09:45:14,780 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/79bff444c4eb480a86111b2bcf54f34d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/79bff444c4eb480a86111b2bcf54f34d 2024-11-14T09:45:14,784 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/C of 29dbb160edffb51f36ebfbb295b932e8 into 79bff444c4eb480a86111b2bcf54f34d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:14,784 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:14,784 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/C, priority=12, startTime=1731577514305; duration=0sec 2024-11-14T09:45:14,784 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:14,784 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:C 2024-11-14T09:45:14,793 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/60ca7a19cc77409da7339cad35ef0781 2024-11-14T09:45:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/1d6b63b4bf2a445c82b83214ba884bf7 is 50, key is test_row_0/B:col10/1731577512663/Put/seqid=0 2024-11-14T09:45:14,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742168_1344 (size=12151) 2024-11-14T09:45:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:14,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577514890, max timestamp=1731577514904Num uniques 859; 2024-11-14T09:45:14,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:14,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577514890, max timestamp=1731577514904Num uniques 859; 2024-11-14T09:45:14,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:14,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731577514890, max timestamp=1731577514904Num uniques 859; 2024-11-14T09:45:14,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T09:45:14,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577574963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577574965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577574966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577575080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577575090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577575090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T09:45:15,213 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/1d6b63b4bf2a445c82b83214ba884bf7 2024-11-14T09:45:15,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/effb1cf86e52471f92d8d23c7d68bedc is 50, key is test_row_0/C:col10/1731577512663/Put/seqid=0 2024-11-14T09:45:15,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742169_1345 (size=12151) 2024-11-14T09:45:15,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577575300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577575300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577575310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577575610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,623 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/effb1cf86e52471f92d8d23c7d68bedc 2024-11-14T09:45:15,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/60ca7a19cc77409da7339cad35ef0781 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/60ca7a19cc77409da7339cad35ef0781 2024-11-14T09:45:15,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577575620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577575630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:15,633 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/60ca7a19cc77409da7339cad35ef0781, entries=150, sequenceid=155, filesize=30.4 K 2024-11-14T09:45:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/1d6b63b4bf2a445c82b83214ba884bf7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1d6b63b4bf2a445c82b83214ba884bf7 2024-11-14T09:45:15,637 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1d6b63b4bf2a445c82b83214ba884bf7, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T09:45:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/effb1cf86e52471f92d8d23c7d68bedc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/effb1cf86e52471f92d8d23c7d68bedc 2024-11-14T09:45:15,641 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/effb1cf86e52471f92d8d23c7d68bedc, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T09:45:15,642 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 29dbb160edffb51f36ebfbb295b932e8 in 1303ms, sequenceid=155, compaction requested=false 2024-11-14T09:45:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-14T09:45:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-14T09:45:15,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-14T09:45:15,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-11-14T09:45:15,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.6150 sec 2024-11-14T09:45:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:16,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-14T09:45:16,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:16,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:16,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:16,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:16,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:16,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:16,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114af1cf624cc50417cad876152d892b611_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577514956/Put/seqid=0 2024-11-14T09:45:16,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742170_1346 (size=14794) 2024-11-14T09:45:16,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577576149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577576150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577576151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T09:45:16,181 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:16,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-14T09:45:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:45:16,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:16,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:16,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577576260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577576270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577576270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:45:16,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:16,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:16,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577576480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577576481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:16,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:16,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577576490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:45:16,537 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:16,540 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114af1cf624cc50417cad876152d892b611_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af1cf624cc50417cad876152d892b611_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:16,541 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5a3fdc374dc84514a0bad8a6a2f43b7e, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:16,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5a3fdc374dc84514a0bad8a6a2f43b7e is 175, key is test_row_0/A:col10/1731577514956/Put/seqid=0 2024-11-14T09:45:16,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742171_1347 (size=39749) 2024-11-14T09:45:16,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:16,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:16,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577576790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577576790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577576801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:45:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45066 deadline: 1731577576930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,937 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 29dbb160edffb51f36ebfbb295b932e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:16,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,946 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5a3fdc374dc84514a0bad8a6a2f43b7e 2024-11-14T09:45:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:16,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1731577576940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:16,949 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 29dbb160edffb51f36ebfbb295b932e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:16,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/f7e08859f4b243b3802070b517a5733b is 50, key is test_row_0/B:col10/1731577514956/Put/seqid=0 2024-11-14T09:45:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742172_1348 (size=12151) 2024-11-14T09:45:17,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:17,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577577301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:17,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577577310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:17,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:17,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:17,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577577310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:45:17,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/f7e08859f4b243b3802070b517a5733b 2024-11-14T09:45:17,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/9c7d6ad6134d4b6c8dd9fd1437b06016 is 50, key is test_row_0/C:col10/1731577514956/Put/seqid=0 2024-11-14T09:45:17,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742173_1349 (size=12151) 2024-11-14T09:45:17,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:17,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/9c7d6ad6134d4b6c8dd9fd1437b06016 2024-11-14T09:45:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5a3fdc374dc84514a0bad8a6a2f43b7e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5a3fdc374dc84514a0bad8a6a2f43b7e 2024-11-14T09:45:17,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5a3fdc374dc84514a0bad8a6a2f43b7e, entries=200, sequenceid=195, filesize=38.8 K 2024-11-14T09:45:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/f7e08859f4b243b3802070b517a5733b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f7e08859f4b243b3802070b517a5733b 2024-11-14T09:45:17,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f7e08859f4b243b3802070b517a5733b, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T09:45:17,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/9c7d6ad6134d4b6c8dd9fd1437b06016 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/9c7d6ad6134d4b6c8dd9fd1437b06016 2024-11-14T09:45:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/9c7d6ad6134d4b6c8dd9fd1437b06016, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T09:45:17,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 29dbb160edffb51f36ebfbb295b932e8 in 1656ms, sequenceid=195, compaction requested=true 2024-11-14T09:45:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:17,783 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:17,783 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:17,784 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:17,784 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:17,784 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/B is initiating minor compaction (all files) 2024-11-14T09:45:17,784 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/A is initiating minor compaction (all files) 2024-11-14T09:45:17,784 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/B in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,784 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/A in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,784 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f415d7d0aba94eb2bbe6b4de5eec216e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1d6b63b4bf2a445c82b83214ba884bf7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f7e08859f4b243b3802070b517a5733b] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=35.8 K 2024-11-14T09:45:17,784 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/d4ca741ee78048b0be2959da3846d1ad, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/60ca7a19cc77409da7339cad35ef0781, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5a3fdc374dc84514a0bad8a6a2f43b7e] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=99.8 K 2024-11-14T09:45:17,784 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,784 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/d4ca741ee78048b0be2959da3846d1ad, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/60ca7a19cc77409da7339cad35ef0781, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5a3fdc374dc84514a0bad8a6a2f43b7e] 2024-11-14T09:45:17,784 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f415d7d0aba94eb2bbe6b4de5eec216e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731577512228 2024-11-14T09:45:17,785 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4ca741ee78048b0be2959da3846d1ad, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731577512228 2024-11-14T09:45:17,785 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d6b63b4bf2a445c82b83214ba884bf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731577512647 2024-11-14T09:45:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,785 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e08859f4b243b3802070b517a5733b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577514956 2024-11-14T09:45:17,785 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ca7a19cc77409da7339cad35ef0781, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731577512647 2024-11-14T09:45:17,785 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a3fdc374dc84514a0bad8a6a2f43b7e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577514956 2024-11-14T09:45:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,791 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,792 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:17,793 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e968dc086dfd4f5296070240fca4dd33_29dbb160edffb51f36ebfbb295b932e8 store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,793 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/05af8e91aa2d41c38633d91334dc898d is 50, key is test_row_0/B:col10/1731577514956/Put/seqid=0 2024-11-14T09:45:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,795 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e968dc086dfd4f5296070240fca4dd33_29dbb160edffb51f36ebfbb295b932e8, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,795 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e968dc086dfd4f5296070240fca4dd33_29dbb160edffb51f36ebfbb295b932e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742174_1350 (size=12493) 2024-11-14T09:45:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742175_1351 (size=4469) 2024-11-14T09:45:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,800 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#A#compaction#295 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,801 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/633dd1e145514faeb5886723fdff78fc is 175, key is test_row_0/A:col10/1731577514956/Put/seqid=0 2024-11-14T09:45:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,804 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/05af8e91aa2d41c38633d91334dc898d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/05af8e91aa2d41c38633d91334dc898d 2024-11-14T09:45:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742176_1352 (size=31447) 2024-11-14T09:45:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,808 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/B of 29dbb160edffb51f36ebfbb295b932e8 into 05af8e91aa2d41c38633d91334dc898d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,808 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:17,808 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/B, priority=13, startTime=1731577517783; duration=0sec 2024-11-14T09:45:17,808 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:17,808 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:B 2024-11-14T09:45:17,808 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,809 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,809 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/C is initiating minor compaction (all files) 2024-11-14T09:45:17,809 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/C in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,809 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/79bff444c4eb480a86111b2bcf54f34d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/effb1cf86e52471f92d8d23c7d68bedc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/9c7d6ad6134d4b6c8dd9fd1437b06016] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=35.8 K 2024-11-14T09:45:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,809 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 79bff444c4eb480a86111b2bcf54f34d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731577512228 2024-11-14T09:45:17,810 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting effb1cf86e52471f92d8d23c7d68bedc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731577512647 2024-11-14T09:45:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,810 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7d6ad6134d4b6c8dd9fd1437b06016, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577514956 2024-11-14T09:45:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,818 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#C#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:17,818 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/325c7f7621f64f9098b230d2945935dc is 50, key is test_row_0/C:col10/1731577514956/Put/seqid=0 2024-11-14T09:45:17,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742177_1353 (size=12493) 2024-11-14T09:45:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,829 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/325c7f7621f64f9098b230d2945935dc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/325c7f7621f64f9098b230d2945935dc 2024-11-14T09:45:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,835 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/C of 29dbb160edffb51f36ebfbb295b932e8 into 325c7f7621f64f9098b230d2945935dc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:17,835 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,835 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/C, priority=13, startTime=1731577517783; duration=0sec 2024-11-14T09:45:17,836 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:17,836 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:C 2024-11-14T09:45:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:45:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,861 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T09:45:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a5dd8f4095b44dfc9a3dc4c08f4e3fdb_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577516149/Put/seqid=0 2024-11-14T09:45:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742178_1354 (size=9814) 2024-11-14T09:45:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,889 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a5dd8f4095b44dfc9a3dc4c08f4e3fdb_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a5dd8f4095b44dfc9a3dc4c08f4e3fdb_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/08ec8797d0a84119a4b2ec9aaa7dedff, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/08ec8797d0a84119a4b2ec9aaa7dedff is 175, key is test_row_0/A:col10/1731577516149/Put/seqid=0 2024-11-14T09:45:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742179_1355 (size=22461) 2024-11-14T09:45:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,209 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/633dd1e145514faeb5886723fdff78fc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/633dd1e145514faeb5886723fdff78fc 2024-11-14T09:45:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,217 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/A of 29dbb160edffb51f36ebfbb295b932e8 into 633dd1e145514faeb5886723fdff78fc(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,217 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:18,217 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/A, priority=13, startTime=1731577517783; duration=0sec 2024-11-14T09:45:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,217 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:18,217 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:A 2024-11-14T09:45:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,310 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/08ec8797d0a84119a4b2ec9aaa7dedff 2024-11-14T09:45:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a26442d83d37486faf94003f47c859a0 is 50, key is test_row_0/B:col10/1731577516149/Put/seqid=0 2024-11-14T09:45:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:45:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742180_1356 (size=9757) 2024-11-14T09:45:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,345 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a26442d83d37486faf94003f47c859a0 2024-11-14T09:45:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/92c713d756064d77a33e9169c8e2dd73 is 50, key is test_row_0/C:col10/1731577516149/Put/seqid=0 2024-11-14T09:45:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742181_1357 (size=9757) 2024-11-14T09:45:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,361 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/92c713d756064d77a33e9169c8e2dd73 2024-11-14T09:45:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:18,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/08ec8797d0a84119a4b2ec9aaa7dedff as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/08ec8797d0a84119a4b2ec9aaa7dedff 2024-11-14T09:45:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,371 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/08ec8797d0a84119a4b2ec9aaa7dedff, entries=100, sequenceid=205, filesize=21.9 K 2024-11-14T09:45:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a26442d83d37486faf94003f47c859a0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a26442d83d37486faf94003f47c859a0 2024-11-14T09:45:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,379 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a26442d83d37486faf94003f47c859a0, entries=100, sequenceid=205, filesize=9.5 K 2024-11-14T09:45:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/92c713d756064d77a33e9169c8e2dd73 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/92c713d756064d77a33e9169c8e2dd73 2024-11-14T09:45:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,386 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/92c713d756064d77a33e9169c8e2dd73, entries=100, sequenceid=205, filesize=9.5 K 2024-11-14T09:45:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,387 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=46.96 KB/48090 for 29dbb160edffb51f36ebfbb295b932e8 in 526ms, sequenceid=205, compaction requested=false 2024-11-14T09:45:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-14T09:45:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-14T09:45:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T09:45:18,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2050 sec 2024-11-14T09:45:18,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2100 sec 2024-11-14T09:45:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:18,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:45:18,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114412483b6908446b6beb53facbfcbe301_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577518399/Put/seqid=0 2024-11-14T09:45:18,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742183_1359 (size=27248) 2024-11-14T09:45:18,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:18,444 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114412483b6908446b6beb53facbfcbe301_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114412483b6908446b6beb53facbfcbe301_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:18,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/0cad06eafde44ce68f03d57d7b282c9d, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:18,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/0cad06eafde44ce68f03d57d7b282c9d is 175, key is test_row_0/A:col10/1731577518399/Put/seqid=0 2024-11-14T09:45:18,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742182_1358 (size=83035) 2024-11-14T09:45:18,450 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/0cad06eafde44ce68f03d57d7b282c9d 2024-11-14T09:45:18,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/078be18e7ce648228f24709a22625c5d is 50, key is test_row_0/B:col10/1731577518399/Put/seqid=0 2024-11-14T09:45:18,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742184_1360 (size=12151) 2024-11-14T09:45:18,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577578464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577578472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577578473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577578580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577578580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577578590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577578790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577578801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577578801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:18,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/078be18e7ce648228f24709a22625c5d 2024-11-14T09:45:18,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308 is 50, key is test_row_0/C:col10/1731577518399/Put/seqid=0 2024-11-14T09:45:18,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742185_1361 (size=12151) 2024-11-14T09:45:19,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577579100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577579110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577579110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308 2024-11-14T09:45:19,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/0cad06eafde44ce68f03d57d7b282c9d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/0cad06eafde44ce68f03d57d7b282c9d 2024-11-14T09:45:19,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/0cad06eafde44ce68f03d57d7b282c9d, entries=450, sequenceid=218, filesize=81.1 K 2024-11-14T09:45:19,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/078be18e7ce648228f24709a22625c5d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/078be18e7ce648228f24709a22625c5d 2024-11-14T09:45:19,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/078be18e7ce648228f24709a22625c5d, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T09:45:19,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308 2024-11-14T09:45:19,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T09:45:19,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 29dbb160edffb51f36ebfbb295b932e8 in 895ms, sequenceid=218, compaction requested=true 2024-11-14T09:45:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:19,295 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:19,295 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:19,296 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:19,296 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:19,296 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/A is initiating minor compaction (all files) 2024-11-14T09:45:19,296 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/B is initiating minor compaction (all files) 2024-11-14T09:45:19,296 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/B in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:19,296 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/A in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:19,296 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/05af8e91aa2d41c38633d91334dc898d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a26442d83d37486faf94003f47c859a0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/078be18e7ce648228f24709a22625c5d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=33.6 K 2024-11-14T09:45:19,296 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/633dd1e145514faeb5886723fdff78fc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/08ec8797d0a84119a4b2ec9aaa7dedff, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/0cad06eafde44ce68f03d57d7b282c9d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=133.7 K 2024-11-14T09:45:19,296 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:19,296 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/633dd1e145514faeb5886723fdff78fc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/08ec8797d0a84119a4b2ec9aaa7dedff, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/0cad06eafde44ce68f03d57d7b282c9d] 2024-11-14T09:45:19,297 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 05af8e91aa2d41c38633d91334dc898d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577514956 2024-11-14T09:45:19,297 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 633dd1e145514faeb5886723fdff78fc, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577514956 2024-11-14T09:45:19,297 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a26442d83d37486faf94003f47c859a0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731577516149 2024-11-14T09:45:19,297 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ec8797d0a84119a4b2ec9aaa7dedff, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731577516149 2024-11-14T09:45:19,298 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 078be18e7ce648228f24709a22625c5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577518376 2024-11-14T09:45:19,298 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cad06eafde44ce68f03d57d7b282c9d, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577518329 2024-11-14T09:45:19,307 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:19,309 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:19,309 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149661fb3174e64c299847d40fc0e1d90c_29dbb160edffb51f36ebfbb295b932e8 store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:19,309 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/50b1eeb5fc9d407fad96c09705922a5a is 50, key is test_row_0/B:col10/1731577518399/Put/seqid=0 2024-11-14T09:45:19,313 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149661fb3174e64c299847d40fc0e1d90c_29dbb160edffb51f36ebfbb295b932e8, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:19,313 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149661fb3174e64c299847d40fc0e1d90c_29dbb160edffb51f36ebfbb295b932e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:19,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742186_1362 (size=12595) 2024-11-14T09:45:19,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742187_1363 (size=4469) 2024-11-14T09:45:19,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T09:45:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:19,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111409cf14aeaae14220983214c8279a1390_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577519625/Put/seqid=0 2024-11-14T09:45:19,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742188_1364 (size=14794) 2024-11-14T09:45:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577579646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577579647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577579651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,718 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/50b1eeb5fc9d407fad96c09705922a5a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/50b1eeb5fc9d407fad96c09705922a5a 2024-11-14T09:45:19,722 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/B of 29dbb160edffb51f36ebfbb295b932e8 into 50b1eeb5fc9d407fad96c09705922a5a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:19,722 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:19,722 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:19,722 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/B, priority=13, startTime=1731577519295; duration=0sec 2024-11-14T09:45:19,722 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:19,723 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:B 2024-11-14T09:45:19,723 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:19,723 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/68c32fcc5ed346bfaa4d4698b175e432 is 175, key is test_row_0/A:col10/1731577518399/Put/seqid=0 2024-11-14T09:45:19,723 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:19,723 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/C is initiating minor compaction (all files) 2024-11-14T09:45:19,723 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/C in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:19,724 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/325c7f7621f64f9098b230d2945935dc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/92c713d756064d77a33e9169c8e2dd73, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=33.6 K 2024-11-14T09:45:19,724 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 325c7f7621f64f9098b230d2945935dc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577514956 2024-11-14T09:45:19,724 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c713d756064d77a33e9169c8e2dd73, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731577516149 2024-11-14T09:45:19,724 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb15e18a6ca4a1ca0efdb9d1f2d5308, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577518376 2024-11-14T09:45:19,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742189_1365 (size=31549) 2024-11-14T09:45:19,730 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:19,730 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/c004250a1db343a5ab7ab6aefd49e27c is 50, key is test_row_0/C:col10/1731577518399/Put/seqid=0 2024-11-14T09:45:19,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742190_1366 (size=12595) 2024-11-14T09:45:19,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577579760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577579770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577579770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577579980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577579980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577579980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:20,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:20,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111409cf14aeaae14220983214c8279a1390_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409cf14aeaae14220983214c8279a1390_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:20,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6708e1accaca49be974d39fb76f4929a, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:20,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6708e1accaca49be974d39fb76f4929a is 175, key is test_row_0/A:col10/1731577519625/Put/seqid=0 2024-11-14T09:45:20,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742191_1367 (size=39749) 2024-11-14T09:45:20,130 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/68c32fcc5ed346bfaa4d4698b175e432 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/68c32fcc5ed346bfaa4d4698b175e432 2024-11-14T09:45:20,134 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/A of 29dbb160edffb51f36ebfbb295b932e8 into 68c32fcc5ed346bfaa4d4698b175e432(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:20,134 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:20,134 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/A, priority=13, startTime=1731577519295; duration=0sec 2024-11-14T09:45:20,135 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:20,135 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:A 2024-11-14T09:45:20,139 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/c004250a1db343a5ab7ab6aefd49e27c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/c004250a1db343a5ab7ab6aefd49e27c 2024-11-14T09:45:20,143 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/C of 29dbb160edffb51f36ebfbb295b932e8 into c004250a1db343a5ab7ab6aefd49e27c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:20,143 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:20,143 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/C, priority=13, startTime=1731577519295; duration=0sec 2024-11-14T09:45:20,144 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:20,144 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:C 2024-11-14T09:45:20,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577580291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577580291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:20,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577580300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:45:20,351 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:20,351 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T09:45:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:45:20,352 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:20,353 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:20,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:20,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6708e1accaca49be974d39fb76f4929a 2024-11-14T09:45:20,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/3c59c8c6652b4d379efa39233a54cfbf is 50, key is test_row_0/B:col10/1731577519625/Put/seqid=0 2024-11-14T09:45:20,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742192_1368 (size=12151) 2024-11-14T09:45:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:45:20,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:45:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:45:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:45:20,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:45:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577580810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577580810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:20,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577580821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:20,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/3c59c8c6652b4d379efa39233a54cfbf 2024-11-14T09:45:20,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/012cc4edff664b8a9b8959bbf6aaea16 is 50, key is test_row_0/C:col10/1731577519625/Put/seqid=0 2024-11-14T09:45:20,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742193_1369 (size=12151) 2024-11-14T09:45:20,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:45:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:20,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:45:21,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:45:21,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:21,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:21,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:21,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:21,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:21,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/012cc4edff664b8a9b8959bbf6aaea16 2024-11-14T09:45:21,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/6708e1accaca49be974d39fb76f4929a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6708e1accaca49be974d39fb76f4929a 2024-11-14T09:45:21,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:45:21,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:21,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:21,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:21,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:21,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:21,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6708e1accaca49be974d39fb76f4929a, entries=200, sequenceid=239, filesize=38.8 K 2024-11-14T09:45:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/3c59c8c6652b4d379efa39233a54cfbf as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/3c59c8c6652b4d379efa39233a54cfbf 2024-11-14T09:45:21,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/3c59c8c6652b4d379efa39233a54cfbf, entries=150, sequenceid=239, filesize=11.9 K 2024-11-14T09:45:21,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/012cc4edff664b8a9b8959bbf6aaea16 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/012cc4edff664b8a9b8959bbf6aaea16 2024-11-14T09:45:21,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/012cc4edff664b8a9b8959bbf6aaea16, entries=150, sequenceid=239, filesize=11.9 K 2024-11-14T09:45:21,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 29dbb160edffb51f36ebfbb295b932e8 in 1658ms, sequenceid=239, compaction requested=false 2024-11-14T09:45:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:21,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:45:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:21,423 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T09:45:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:21,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e169d2d6b8c14bbfa7aee68359c662c1_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577519645/Put/seqid=0 2024-11-14T09:45:21,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742194_1370 (size=12304) 2024-11-14T09:45:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:45:21,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:21,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:21,842 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e169d2d6b8c14bbfa7aee68359c662c1_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e169d2d6b8c14bbfa7aee68359c662c1_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/aa1eebf150cb43e2969c985cde8e233e, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/aa1eebf150cb43e2969c985cde8e233e is 175, key is test_row_0/A:col10/1731577519645/Put/seqid=0 2024-11-14T09:45:21,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742195_1371 (size=31105) 2024-11-14T09:45:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731577521854, max timestamp=1731577521880Num uniques 859; 2024-11-14T09:45:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:45:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731577521854, max timestamp=1731577521880Num uniques 859; 2024-11-14T09:45:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731577521854, max timestamp=1731577521880Num uniques 859; 2024-11-14T09:45:21,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:45:21,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T09:45:21,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577581928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:21,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:21,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577581953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577581953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577582060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577582080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577582080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,253 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/aa1eebf150cb43e2969c985cde8e233e 2024-11-14T09:45:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577582281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577582300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577582300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/32db9579a524498aaffcec3a32073fa9 is 50, key is test_row_0/B:col10/1731577519645/Put/seqid=0 2024-11-14T09:45:22,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742196_1372 (size=12151) 2024-11-14T09:45:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:45:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577582600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577582611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577582611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:22,725 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/32db9579a524498aaffcec3a32073fa9 2024-11-14T09:45:22,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/33f6ebcd8c3b442cb50b65411452a9f6 is 50, key is test_row_0/C:col10/1731577519645/Put/seqid=0 2024-11-14T09:45:22,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742197_1373 (size=12151) 2024-11-14T09:45:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577583120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:23,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577583120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577583130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:23,282 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/33f6ebcd8c3b442cb50b65411452a9f6 2024-11-14T09:45:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/aa1eebf150cb43e2969c985cde8e233e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/aa1eebf150cb43e2969c985cde8e233e 2024-11-14T09:45:23,289 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/aa1eebf150cb43e2969c985cde8e233e, entries=150, sequenceid=253, filesize=30.4 K 2024-11-14T09:45:23,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/32db9579a524498aaffcec3a32073fa9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/32db9579a524498aaffcec3a32073fa9 2024-11-14T09:45:23,292 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/32db9579a524498aaffcec3a32073fa9, entries=150, sequenceid=253, filesize=11.9 K 2024-11-14T09:45:23,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/33f6ebcd8c3b442cb50b65411452a9f6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/33f6ebcd8c3b442cb50b65411452a9f6 2024-11-14T09:45:23,296 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/33f6ebcd8c3b442cb50b65411452a9f6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-14T09:45:23,297 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 29dbb160edffb51f36ebfbb295b932e8 in 1874ms, sequenceid=253, compaction requested=true 2024-11-14T09:45:23,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:23,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:23,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T09:45:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T09:45:23,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T09:45:23,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9450 sec 2024-11-14T09:45:23,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.9490 sec 2024-11-14T09:45:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:24,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-14T09:45:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:24,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:24,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f66c224e5d694839965fd1414f2023a6_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577524147/Put/seqid=0 2024-11-14T09:45:24,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742198_1374 (size=14994) 2024-11-14T09:45:24,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:24,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f66c224e5d694839965fd1414f2023a6_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f66c224e5d694839965fd1414f2023a6_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:24,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/e90b0e4bc9ef4316869444beacda721c, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:24,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/e90b0e4bc9ef4316869444beacda721c is 175, key is test_row_0/A:col10/1731577524147/Put/seqid=0 2024-11-14T09:45:24,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742199_1375 (size=39949) 2024-11-14T09:45:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577584161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577584165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577584166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577584280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577584281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577584290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577584500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577584500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577584501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:45:24,520 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:24,520 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T09:45:24,522 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:45:24,522 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:24,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:24,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/e90b0e4bc9ef4316869444beacda721c 2024-11-14T09:45:24,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a93eb17b63b541ec9ec8b6affa77663f is 50, key is test_row_0/B:col10/1731577524147/Put/seqid=0 2024-11-14T09:45:24,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742200_1376 (size=12301) 2024-11-14T09:45:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:45:24,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:45:24,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:24,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:24,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:24,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577584813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577584813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577584813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:24,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:45:24,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:24,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:24,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:24,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:45:24,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a93eb17b63b541ec9ec8b6affa77663f 2024-11-14T09:45:24,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:45:24,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:24,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:24,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:24,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:24,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/e054a7316718452a8e66ed145617dcc4 is 50, key is test_row_0/C:col10/1731577524147/Put/seqid=0 2024-11-14T09:45:24,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742201_1377 (size=12301) 2024-11-14T09:45:25,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:45:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:45:25,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:45:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:25,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45122 deadline: 1731577585320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:25,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1731577585330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:25,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731577585330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:25,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:45:25,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/e054a7316718452a8e66ed145617dcc4 2024-11-14T09:45:25,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/e90b0e4bc9ef4316869444beacda721c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/e90b0e4bc9ef4316869444beacda721c 2024-11-14T09:45:25,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/e90b0e4bc9ef4316869444beacda721c, entries=200, sequenceid=289, filesize=39.0 K 2024-11-14T09:45:25,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a93eb17b63b541ec9ec8b6affa77663f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a93eb17b63b541ec9ec8b6affa77663f 2024-11-14T09:45:25,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a93eb17b63b541ec9ec8b6affa77663f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-14T09:45:25,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/e054a7316718452a8e66ed145617dcc4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/e054a7316718452a8e66ed145617dcc4 2024-11-14T09:45:25,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/e054a7316718452a8e66ed145617dcc4, entries=150, sequenceid=289, filesize=12.0 K 2024-11-14T09:45:25,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 29dbb160edffb51f36ebfbb295b932e8 in 1266ms, sequenceid=289, compaction requested=true 2024-11-14T09:45:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:25,414 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:25,414 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 29dbb160edffb51f36ebfbb295b932e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:25,415 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:25,415 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142352 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:25,415 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/A is initiating minor compaction (all files) 2024-11-14T09:45:25,415 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/B is initiating minor compaction (all files) 2024-11-14T09:45:25,415 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/B in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,415 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/A in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,416 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/50b1eeb5fc9d407fad96c09705922a5a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/3c59c8c6652b4d379efa39233a54cfbf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/32db9579a524498aaffcec3a32073fa9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a93eb17b63b541ec9ec8b6affa77663f] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=48.0 K 2024-11-14T09:45:25,416 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/68c32fcc5ed346bfaa4d4698b175e432, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6708e1accaca49be974d39fb76f4929a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/aa1eebf150cb43e2969c985cde8e233e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/e90b0e4bc9ef4316869444beacda721c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=139.0 K 2024-11-14T09:45:25,416 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,416 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/68c32fcc5ed346bfaa4d4698b175e432, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6708e1accaca49be974d39fb76f4929a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/aa1eebf150cb43e2969c985cde8e233e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/e90b0e4bc9ef4316869444beacda721c] 2024-11-14T09:45:25,416 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b1eeb5fc9d407fad96c09705922a5a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577518376 2024-11-14T09:45:25,416 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c32fcc5ed346bfaa4d4698b175e432, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577518376 2024-11-14T09:45:25,416 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c59c8c6652b4d379efa39233a54cfbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731577518449 2024-11-14T09:45:25,417 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6708e1accaca49be974d39fb76f4929a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731577518449 2024-11-14T09:45:25,417 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 32db9579a524498aaffcec3a32073fa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731577519640 2024-11-14T09:45:25,417 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa1eebf150cb43e2969c985cde8e233e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731577519640 2024-11-14T09:45:25,417 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a93eb17b63b541ec9ec8b6affa77663f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731577521926 2024-11-14T09:45:25,418 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e90b0e4bc9ef4316869444beacda721c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731577521905 2024-11-14T09:45:25,424 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:25,425 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:25,426 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/973a9b916b404c70b2a43cfa1e2931f2 is 50, key is test_row_0/B:col10/1731577524147/Put/seqid=0 2024-11-14T09:45:25,429 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145cf73f47f007424790e1375c26367f58_29dbb160edffb51f36ebfbb295b932e8 store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:25,432 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145cf73f47f007424790e1375c26367f58_29dbb160edffb51f36ebfbb295b932e8, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:25,432 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145cf73f47f007424790e1375c26367f58_29dbb160edffb51f36ebfbb295b932e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:25,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742202_1378 (size=12881) 2024-11-14T09:45:25,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742203_1379 (size=4469) 2024-11-14T09:45:25,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:45:25,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,439 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T09:45:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:25,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a204d7cca7494aa98a75483e4656ad09_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577524160/Put/seqid=0 2024-11-14T09:45:25,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742204_1380 (size=12454) 2024-11-14T09:45:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:45:25,711 INFO [Thread-1441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1441. 2024-11-14T09:45:25,711 DEBUG [Thread-1441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:25,711 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,711 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:25,714 INFO [Thread-1443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1443. 2024-11-14T09:45:25,714 DEBUG [Thread-1443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:25,714 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,714 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:25,716 INFO [Thread-1437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1437. 2024-11-14T09:45:25,716 DEBUG [Thread-1437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:25,716 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,716 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:25,719 INFO [Thread-1435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1435. 2024-11-14T09:45:25,719 DEBUG [Thread-1435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:25,719 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,719 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:25,721 INFO [Thread-1439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1439. 2024-11-14T09:45:25,721 DEBUG [Thread-1439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:25,721 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,721 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:25,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:25,842 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:25,842 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/1710f6c73f43428791b57c8758d0b45a is 175, key is test_row_0/A:col10/1731577524147/Put/seqid=0 2024-11-14T09:45:25,845 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/973a9b916b404c70b2a43cfa1e2931f2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/973a9b916b404c70b2a43cfa1e2931f2 2024-11-14T09:45:25,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742205_1381 (size=31835) 2024-11-14T09:45:25,850 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/B of 29dbb160edffb51f36ebfbb295b932e8 into 973a9b916b404c70b2a43cfa1e2931f2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:25,850 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:25,850 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/B, priority=12, startTime=1731577525414; duration=0sec 2024-11-14T09:45:25,851 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:25,851 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:B 2024-11-14T09:45:25,851 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:25,851 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/1710f6c73f43428791b57c8758d0b45a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/1710f6c73f43428791b57c8758d0b45a 2024-11-14T09:45:25,852 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:25,852 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 29dbb160edffb51f36ebfbb295b932e8/C is initiating minor compaction (all files) 2024-11-14T09:45:25,852 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 29dbb160edffb51f36ebfbb295b932e8/C in TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:25,852 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/c004250a1db343a5ab7ab6aefd49e27c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/012cc4edff664b8a9b8959bbf6aaea16, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/33f6ebcd8c3b442cb50b65411452a9f6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/e054a7316718452a8e66ed145617dcc4] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp, totalSize=48.0 K 2024-11-14T09:45:25,852 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c004250a1db343a5ab7ab6aefd49e27c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577518376 2024-11-14T09:45:25,852 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 012cc4edff664b8a9b8959bbf6aaea16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731577518449 2024-11-14T09:45:25,852 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 33f6ebcd8c3b442cb50b65411452a9f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731577519640 2024-11-14T09:45:25,853 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e054a7316718452a8e66ed145617dcc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731577521926 2024-11-14T09:45:25,855 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/A of 29dbb160edffb51f36ebfbb295b932e8 into 1710f6c73f43428791b57c8758d0b45a(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:25,855 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:25,855 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/A, priority=12, startTime=1731577525414; duration=0sec 2024-11-14T09:45:25,855 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:25,855 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:A 2024-11-14T09:45:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:25,860 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 29dbb160edffb51f36ebfbb295b932e8#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:25,860 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a204d7cca7494aa98a75483e4656ad09_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a204d7cca7494aa98a75483e4656ad09_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:25,860 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/31bd54bbbca349aba6214dbcec1e770b is 50, key is test_row_0/C:col10/1731577524147/Put/seqid=0 2024-11-14T09:45:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5b46c821df5b4cf1adf641b7c760ea28, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5b46c821df5b4cf1adf641b7c760ea28 is 175, key is test_row_0/A:col10/1731577524160/Put/seqid=0 2024-11-14T09:45:25,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742206_1382 (size=12881) 2024-11-14T09:45:25,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742207_1383 (size=31255) 2024-11-14T09:45:26,265 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5b46c821df5b4cf1adf641b7c760ea28 2024-11-14T09:45:26,273 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/31bd54bbbca349aba6214dbcec1e770b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/31bd54bbbca349aba6214dbcec1e770b 2024-11-14T09:45:26,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/285e13304ad8477bb05551a2455d2bc3 is 50, key is test_row_0/B:col10/1731577524160/Put/seqid=0 2024-11-14T09:45:26,280 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 29dbb160edffb51f36ebfbb295b932e8/C of 29dbb160edffb51f36ebfbb295b932e8 into 31bd54bbbca349aba6214dbcec1e770b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:26,280 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:26,280 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., storeName=29dbb160edffb51f36ebfbb295b932e8/C, priority=12, startTime=1731577525414; duration=0sec 2024-11-14T09:45:26,280 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:26,280 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 29dbb160edffb51f36ebfbb295b932e8:C 2024-11-14T09:45:26,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742208_1384 (size=12301) 2024-11-14T09:45:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:26,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. as already flushing 2024-11-14T09:45:26,344 INFO [Thread-1428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1428. 2024-11-14T09:45:26,344 INFO [Thread-1432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1432. 2024-11-14T09:45:26,344 INFO [Thread-1424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1424. 2024-11-14T09:45:26,344 DEBUG [Thread-1428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:26,344 DEBUG [Thread-1432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:26,344 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:26,344 DEBUG [Thread-1424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:26,344 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:26,344 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:26,344 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:26,344 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:26,344 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:26,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:26,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:26,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:26,643 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:45:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:45:26,682 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/285e13304ad8477bb05551a2455d2bc3 2024-11-14T09:45:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/ded62172626d4d639653c8b65c2040fc is 50, key is test_row_0/C:col10/1731577524160/Put/seqid=0 2024-11-14T09:45:26,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742209_1385 (size=12301) 2024-11-14T09:45:27,004 INFO [Thread-1430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1430. 2024-11-14T09:45:27,004 DEBUG [Thread-1430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:27,004 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:27,005 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:27,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:27,043 INFO [Thread-1426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1426. 2024-11-14T09:45:27,043 DEBUG [Thread-1426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:27,043 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:27,044 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:27,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:27,099 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/ded62172626d4d639653c8b65c2040fc 2024-11-14T09:45:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/5b46c821df5b4cf1adf641b7c760ea28 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5b46c821df5b4cf1adf641b7c760ea28 2024-11-14T09:45:27,111 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5b46c821df5b4cf1adf641b7c760ea28, entries=150, sequenceid=298, filesize=30.5 K 2024-11-14T09:45:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/285e13304ad8477bb05551a2455d2bc3 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/285e13304ad8477bb05551a2455d2bc3 2024-11-14T09:45:27,116 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/285e13304ad8477bb05551a2455d2bc3, entries=150, sequenceid=298, filesize=12.0 K 2024-11-14T09:45:27,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/ded62172626d4d639653c8b65c2040fc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ded62172626d4d639653c8b65c2040fc 2024-11-14T09:45:27,120 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ded62172626d4d639653c8b65c2040fc, entries=150, sequenceid=298, filesize=12.0 K 2024-11-14T09:45:27,121 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=33.54 KB/34350 for 29dbb160edffb51f36ebfbb295b932e8 in 1683ms, sequenceid=298, compaction requested=false 2024-11-14T09:45:27,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 29dbb160edffb51f36ebfbb295b932e8: 2024-11-14T09:45:27,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:27,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T09:45:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T09:45:27,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T09:45:27,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6010 sec 2024-11-14T09:45:27,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.6040 sec 2024-11-14T09:45:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:45:28,702 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1131 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3391 rows 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-14T09:45:28,702 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3383 rows 2024-11-14T09:45:28,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1136 2024-11-14T09:45:28,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3408 rows 2024-11-14T09:45:28,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1135 2024-11-14T09:45:28,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3393 rows 2024-11-14T09:45:28,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1140 2024-11-14T09:45:28,703 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3408 rows 2024-11-14T09:45:28,703 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:45:28,703 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:45:28,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:28,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:28,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:28,706 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:45:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:45:28,712 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577528712"}]},"ts":"1731577528712"} 2024-11-14T09:45:28,713 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:45:28,713 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:45:28,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:45:28,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, UNASSIGN}] 2024-11-14T09:45:28,716 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, UNASSIGN 2024-11-14T09:45:28,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:28,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, UNASSIGN because future has completed 2024-11-14T09:45:28,718 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:45:28,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:45:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:45:28,871 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(122): Close 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:28,871 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:45:28,872 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1722): Closing 29dbb160edffb51f36ebfbb295b932e8, disabling compactions & flushes 2024-11-14T09:45:28,872 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:28,872 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:28,872 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. after waiting 0 ms 2024-11-14T09:45:28,872 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:28,872 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(2902): Flushing 29dbb160edffb51f36ebfbb295b932e8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T09:45:28,873 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=A 2024-11-14T09:45:28,873 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:28,873 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=B 2024-11-14T09:45:28,873 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:28,873 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 29dbb160edffb51f36ebfbb295b932e8, store=C 2024-11-14T09:45:28,874 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:28,883 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b3b7284d92446f7ad4c0903108d694f_29dbb160edffb51f36ebfbb295b932e8 is 50, key is test_row_0/A:col10/1731577526341/Put/seqid=0 2024-11-14T09:45:28,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742210_1386 (size=12454) 2024-11-14T09:45:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:45:29,289 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:29,299 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147b3b7284d92446f7ad4c0903108d694f_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b3b7284d92446f7ad4c0903108d694f_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:29,300 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/be19ecb069d64633bed461808e7828e0, store: [table=TestAcidGuarantees family=A region=29dbb160edffb51f36ebfbb295b932e8] 2024-11-14T09:45:29,301 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/be19ecb069d64633bed461808e7828e0 is 175, key is test_row_0/A:col10/1731577526341/Put/seqid=0 2024-11-14T09:45:29,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742211_1387 (size=31255) 2024-11-14T09:45:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:45:29,706 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/be19ecb069d64633bed461808e7828e0 2024-11-14T09:45:29,719 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a788396f971e436f8f990dd577ca6874 is 50, key is test_row_0/B:col10/1731577526341/Put/seqid=0 2024-11-14T09:45:29,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742212_1388 (size=12301) 2024-11-14T09:45:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:45:30,124 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a788396f971e436f8f990dd577ca6874 2024-11-14T09:45:30,137 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/ecdd1b635f14455fbd83ed95bd3ae9f0 is 50, key is test_row_0/C:col10/1731577526341/Put/seqid=0 2024-11-14T09:45:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742213_1389 (size=12301) 2024-11-14T09:45:30,543 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/ecdd1b635f14455fbd83ed95bd3ae9f0 2024-11-14T09:45:30,552 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/A/be19ecb069d64633bed461808e7828e0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/be19ecb069d64633bed461808e7828e0 2024-11-14T09:45:30,557 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/be19ecb069d64633bed461808e7828e0, entries=150, sequenceid=309, filesize=30.5 K 2024-11-14T09:45:30,558 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/B/a788396f971e436f8f990dd577ca6874 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a788396f971e436f8f990dd577ca6874 2024-11-14T09:45:30,562 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a788396f971e436f8f990dd577ca6874, entries=150, sequenceid=309, filesize=12.0 K 2024-11-14T09:45:30,563 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/.tmp/C/ecdd1b635f14455fbd83ed95bd3ae9f0 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ecdd1b635f14455fbd83ed95bd3ae9f0 2024-11-14T09:45:30,567 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ecdd1b635f14455fbd83ed95bd3ae9f0, entries=150, sequenceid=309, filesize=12.0 K 2024-11-14T09:45:30,568 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 29dbb160edffb51f36ebfbb295b932e8 in 1696ms, sequenceid=309, compaction requested=true 2024-11-14T09:45:30,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/b4d7a126b72f40f09d468d927590449b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/9ba9852f9dda43aead9854540519b9af, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6330d6583ea2427aa8035f451fd4b19d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5f6882b306c046caa8cb1e3de1843329, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/31ce95e891c64b159bead14fac4e835f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/ad9bc55e8cd046a483a902d6dc0bc004, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/d4ca741ee78048b0be2959da3846d1ad, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/60ca7a19cc77409da7339cad35ef0781, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5a3fdc374dc84514a0bad8a6a2f43b7e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/633dd1e145514faeb5886723fdff78fc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/08ec8797d0a84119a4b2ec9aaa7dedff, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/0cad06eafde44ce68f03d57d7b282c9d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/68c32fcc5ed346bfaa4d4698b175e432, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6708e1accaca49be974d39fb76f4929a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/aa1eebf150cb43e2969c985cde8e233e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/e90b0e4bc9ef4316869444beacda721c] to archive 2024-11-14T09:45:30,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:45:30,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/b4d7a126b72f40f09d468d927590449b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/b4d7a126b72f40f09d468d927590449b 2024-11-14T09:45:30,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/9ba9852f9dda43aead9854540519b9af to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/9ba9852f9dda43aead9854540519b9af 2024-11-14T09:45:30,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6330d6583ea2427aa8035f451fd4b19d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6330d6583ea2427aa8035f451fd4b19d 2024-11-14T09:45:30,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5f6882b306c046caa8cb1e3de1843329 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5f6882b306c046caa8cb1e3de1843329 2024-11-14T09:45:30,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/31ce95e891c64b159bead14fac4e835f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/31ce95e891c64b159bead14fac4e835f 2024-11-14T09:45:30,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/27fe3d1f7ef7447ca0f3f9ba0eeaa0f1 2024-11-14T09:45:30,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/ad9bc55e8cd046a483a902d6dc0bc004 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/ad9bc55e8cd046a483a902d6dc0bc004 2024-11-14T09:45:30,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/d4ca741ee78048b0be2959da3846d1ad to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/d4ca741ee78048b0be2959da3846d1ad 2024-11-14T09:45:30,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/60ca7a19cc77409da7339cad35ef0781 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/60ca7a19cc77409da7339cad35ef0781 2024-11-14T09:45:30,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5a3fdc374dc84514a0bad8a6a2f43b7e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5a3fdc374dc84514a0bad8a6a2f43b7e 2024-11-14T09:45:30,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/633dd1e145514faeb5886723fdff78fc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/633dd1e145514faeb5886723fdff78fc 2024-11-14T09:45:30,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/08ec8797d0a84119a4b2ec9aaa7dedff to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/08ec8797d0a84119a4b2ec9aaa7dedff 2024-11-14T09:45:30,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/0cad06eafde44ce68f03d57d7b282c9d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/0cad06eafde44ce68f03d57d7b282c9d 2024-11-14T09:45:30,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/68c32fcc5ed346bfaa4d4698b175e432 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/68c32fcc5ed346bfaa4d4698b175e432 2024-11-14T09:45:30,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6708e1accaca49be974d39fb76f4929a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/6708e1accaca49be974d39fb76f4929a 2024-11-14T09:45:30,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/aa1eebf150cb43e2969c985cde8e233e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/aa1eebf150cb43e2969c985cde8e233e 2024-11-14T09:45:30,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/e90b0e4bc9ef4316869444beacda721c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/e90b0e4bc9ef4316869444beacda721c 2024-11-14T09:45:30,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/867181ccbcca416ca0f7668a36651960, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/8ff4fa692887412fbf1e77da28028cf6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/b09e4aae92fb4b6ebe41a2b0aa5b32b8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/c35e19ccef754a6d9c6ab6809bdf11f2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1c70d9b3520745c1ab02f773c231dac7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/35fa1d366c854b179b6616faaeacee29, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f415d7d0aba94eb2bbe6b4de5eec216e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/0c8ac46825e6485491e198cc1d80c13f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1d6b63b4bf2a445c82b83214ba884bf7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/05af8e91aa2d41c38633d91334dc898d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f7e08859f4b243b3802070b517a5733b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a26442d83d37486faf94003f47c859a0, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/50b1eeb5fc9d407fad96c09705922a5a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/078be18e7ce648228f24709a22625c5d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/3c59c8c6652b4d379efa39233a54cfbf, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/32db9579a524498aaffcec3a32073fa9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a93eb17b63b541ec9ec8b6affa77663f] to archive 2024-11-14T09:45:30,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:45:30,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/867181ccbcca416ca0f7668a36651960 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/867181ccbcca416ca0f7668a36651960 2024-11-14T09:45:30,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/8ff4fa692887412fbf1e77da28028cf6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/8ff4fa692887412fbf1e77da28028cf6 2024-11-14T09:45:30,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/b09e4aae92fb4b6ebe41a2b0aa5b32b8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/b09e4aae92fb4b6ebe41a2b0aa5b32b8 2024-11-14T09:45:30,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/c35e19ccef754a6d9c6ab6809bdf11f2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/c35e19ccef754a6d9c6ab6809bdf11f2 2024-11-14T09:45:30,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1c70d9b3520745c1ab02f773c231dac7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1c70d9b3520745c1ab02f773c231dac7 2024-11-14T09:45:30,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/35fa1d366c854b179b6616faaeacee29 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/35fa1d366c854b179b6616faaeacee29 2024-11-14T09:45:30,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f415d7d0aba94eb2bbe6b4de5eec216e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f415d7d0aba94eb2bbe6b4de5eec216e 2024-11-14T09:45:30,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/0c8ac46825e6485491e198cc1d80c13f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/0c8ac46825e6485491e198cc1d80c13f 2024-11-14T09:45:30,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1d6b63b4bf2a445c82b83214ba884bf7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/1d6b63b4bf2a445c82b83214ba884bf7 2024-11-14T09:45:30,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/05af8e91aa2d41c38633d91334dc898d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/05af8e91aa2d41c38633d91334dc898d 2024-11-14T09:45:30,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f7e08859f4b243b3802070b517a5733b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/f7e08859f4b243b3802070b517a5733b 2024-11-14T09:45:30,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a26442d83d37486faf94003f47c859a0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a26442d83d37486faf94003f47c859a0 2024-11-14T09:45:30,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/50b1eeb5fc9d407fad96c09705922a5a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/50b1eeb5fc9d407fad96c09705922a5a 2024-11-14T09:45:30,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/078be18e7ce648228f24709a22625c5d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/078be18e7ce648228f24709a22625c5d 2024-11-14T09:45:30,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/3c59c8c6652b4d379efa39233a54cfbf to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/3c59c8c6652b4d379efa39233a54cfbf 2024-11-14T09:45:30,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/32db9579a524498aaffcec3a32073fa9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/32db9579a524498aaffcec3a32073fa9 2024-11-14T09:45:30,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a93eb17b63b541ec9ec8b6affa77663f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a93eb17b63b541ec9ec8b6affa77663f 2024-11-14T09:45:30,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/35e45773cfee4a51ac1462d05306e477, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/2cf6f15c81dd42da9661e0697dfbf1dc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/185a1d7054314e4c9d8bc681968defb7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/309f9e8b6e4c41589bc04a9b239caf37, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/a06fede6b96d43f5a97ed10d32510c3c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/98fdf6c582944d698f7357f460cc7582, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/79bff444c4eb480a86111b2bcf54f34d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/34b2fea56a7d46d8b57e22dfd9a90fb6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/effb1cf86e52471f92d8d23c7d68bedc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/325c7f7621f64f9098b230d2945935dc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/9c7d6ad6134d4b6c8dd9fd1437b06016, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/92c713d756064d77a33e9169c8e2dd73, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/c004250a1db343a5ab7ab6aefd49e27c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/012cc4edff664b8a9b8959bbf6aaea16, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/33f6ebcd8c3b442cb50b65411452a9f6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/e054a7316718452a8e66ed145617dcc4] to archive 2024-11-14T09:45:30,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:45:30,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/35e45773cfee4a51ac1462d05306e477 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/35e45773cfee4a51ac1462d05306e477 2024-11-14T09:45:30,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/2cf6f15c81dd42da9661e0697dfbf1dc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/2cf6f15c81dd42da9661e0697dfbf1dc 2024-11-14T09:45:30,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/185a1d7054314e4c9d8bc681968defb7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/185a1d7054314e4c9d8bc681968defb7 2024-11-14T09:45:30,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/309f9e8b6e4c41589bc04a9b239caf37 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/309f9e8b6e4c41589bc04a9b239caf37 2024-11-14T09:45:30,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/a06fede6b96d43f5a97ed10d32510c3c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/a06fede6b96d43f5a97ed10d32510c3c 2024-11-14T09:45:30,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/98fdf6c582944d698f7357f460cc7582 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/98fdf6c582944d698f7357f460cc7582 2024-11-14T09:45:30,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/79bff444c4eb480a86111b2bcf54f34d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/79bff444c4eb480a86111b2bcf54f34d 2024-11-14T09:45:30,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/34b2fea56a7d46d8b57e22dfd9a90fb6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/34b2fea56a7d46d8b57e22dfd9a90fb6 2024-11-14T09:45:30,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/effb1cf86e52471f92d8d23c7d68bedc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/effb1cf86e52471f92d8d23c7d68bedc 2024-11-14T09:45:30,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/325c7f7621f64f9098b230d2945935dc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/325c7f7621f64f9098b230d2945935dc 2024-11-14T09:45:30,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/9c7d6ad6134d4b6c8dd9fd1437b06016 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/9c7d6ad6134d4b6c8dd9fd1437b06016 2024-11-14T09:45:30,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/92c713d756064d77a33e9169c8e2dd73 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/92c713d756064d77a33e9169c8e2dd73 2024-11-14T09:45:30,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/c004250a1db343a5ab7ab6aefd49e27c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/c004250a1db343a5ab7ab6aefd49e27c 2024-11-14T09:45:30,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/0bb15e18a6ca4a1ca0efdb9d1f2d5308 2024-11-14T09:45:30,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/012cc4edff664b8a9b8959bbf6aaea16 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/012cc4edff664b8a9b8959bbf6aaea16 2024-11-14T09:45:30,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/33f6ebcd8c3b442cb50b65411452a9f6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/33f6ebcd8c3b442cb50b65411452a9f6 2024-11-14T09:45:30,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/e054a7316718452a8e66ed145617dcc4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/e054a7316718452a8e66ed145617dcc4 2024-11-14T09:45:30,620 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/recovered.edits/312.seqid, newMaxSeqId=312, maxSeqId=4 2024-11-14T09:45:30,621 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. 2024-11-14T09:45:30,621 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1676): Region close journal for 29dbb160edffb51f36ebfbb295b932e8: Waiting for close lock at 1731577528871Running coprocessor pre-close hooks at 1731577528871Disabling compacts and flushes for region at 1731577528871Disabling writes for close at 1731577528872 (+1 ms)Obtaining lock to block concurrent updates at 1731577528872Preparing flush snapshotting stores in 29dbb160edffb51f36ebfbb295b932e8 at 1731577528872Finished memstore snapshotting TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731577528874 (+2 ms)Flushing stores of TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8. at 1731577528875 (+1 ms)Flushing 29dbb160edffb51f36ebfbb295b932e8/A: creating writer at 1731577528875Flushing mob file 29dbb160edffb51f36ebfbb295b932e8/A: appending metadata at 1731577528882 (+7 ms)Flushing mob file 29dbb160edffb51f36ebfbb295b932e8/A: closing flushed file at 1731577528882Flushing 29dbb160edffb51f36ebfbb295b932e8/A: appending metadata at 1731577529300 (+418 ms)Flushing 29dbb160edffb51f36ebfbb295b932e8/A: closing flushed file at 1731577529300Flushing 29dbb160edffb51f36ebfbb295b932e8/B: creating writer at 1731577529715 (+415 ms)Flushing 29dbb160edffb51f36ebfbb295b932e8/B: appending metadata at 1731577529718 (+3 ms)Flushing 29dbb160edffb51f36ebfbb295b932e8/B: closing flushed file at 1731577529718Flushing 29dbb160edffb51f36ebfbb295b932e8/C: creating writer at 1731577530133 (+415 ms)Flushing 29dbb160edffb51f36ebfbb295b932e8/C: appending metadata at 1731577530136 (+3 ms)Flushing 29dbb160edffb51f36ebfbb295b932e8/C: closing flushed file at 1731577530136Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fd3b381: reopening flushed file at 1731577530552 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a04cc2f: reopening flushed file at 1731577530558 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73ea802d: reopening flushed file at 1731577530562 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 29dbb160edffb51f36ebfbb295b932e8 in 1696ms, sequenceid=309, compaction requested=true at 1731577530568 (+6 ms)Writing region close event to WAL at 1731577530618 (+50 ms)Running coprocessor post-close hooks at 1731577530620 (+2 ms)Closed at 1731577530620 2024-11-14T09:45:30,622 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(157): Closed 29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,623 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=29dbb160edffb51f36ebfbb295b932e8, regionState=CLOSED 2024-11-14T09:45:30,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:45:30,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T09:45:30,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseRegionProcedure 29dbb160edffb51f36ebfbb295b932e8, server=defc576eb6b7,42511,1731577408298 in 1.9060 sec 2024-11-14T09:45:30,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-14T09:45:30,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=29dbb160edffb51f36ebfbb295b932e8, UNASSIGN in 1.9110 sec 2024-11-14T09:45:30,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T09:45:30,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9140 sec 2024-11-14T09:45:30,630 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577530630"}]},"ts":"1731577530630"} 2024-11-14T09:45:30,631 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:45:30,631 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:45:30,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9250 sec 2024-11-14T09:45:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:45:30,871 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:30,872 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:45:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:30,875 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:30,876 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=103, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:45:30,879 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,884 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/recovered.edits] 2024-11-14T09:45:30,888 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/1710f6c73f43428791b57c8758d0b45a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/1710f6c73f43428791b57c8758d0b45a 2024-11-14T09:45:30,890 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5b46c821df5b4cf1adf641b7c760ea28 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/5b46c821df5b4cf1adf641b7c760ea28 2024-11-14T09:45:30,891 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/be19ecb069d64633bed461808e7828e0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/A/be19ecb069d64633bed461808e7828e0 2024-11-14T09:45:30,894 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/285e13304ad8477bb05551a2455d2bc3 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/285e13304ad8477bb05551a2455d2bc3 2024-11-14T09:45:30,896 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/973a9b916b404c70b2a43cfa1e2931f2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/973a9b916b404c70b2a43cfa1e2931f2 2024-11-14T09:45:30,897 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a788396f971e436f8f990dd577ca6874 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/B/a788396f971e436f8f990dd577ca6874 2024-11-14T09:45:30,898 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/31bd54bbbca349aba6214dbcec1e770b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/31bd54bbbca349aba6214dbcec1e770b 2024-11-14T09:45:30,899 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ded62172626d4d639653c8b65c2040fc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ded62172626d4d639653c8b65c2040fc 2024-11-14T09:45:30,900 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ecdd1b635f14455fbd83ed95bd3ae9f0 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/C/ecdd1b635f14455fbd83ed95bd3ae9f0 2024-11-14T09:45:30,902 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/recovered.edits/312.seqid to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8/recovered.edits/312.seqid 2024-11-14T09:45:30,902 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,902 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:45:30,902 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:45:30,903 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:45:30,905 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409cf14aeaae14220983214c8279a1390_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409cf14aeaae14220983214c8279a1390_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,906 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142d70919d630b4c78b51e72b63983ac24_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142d70919d630b4c78b51e72b63983ac24_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,907 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142ff4cec269464c79943919baf7ef66cb_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142ff4cec269464c79943919baf7ef66cb_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,908 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114412483b6908446b6beb53facbfcbe301_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114412483b6908446b6beb53facbfcbe301_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,908 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111442cd9bf286b44bb39079dc277f891326_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111442cd9bf286b44bb39079dc277f891326_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,909 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d0e976a30ba4cc8a7b92fd72ba18ebf_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146d0e976a30ba4cc8a7b92fd72ba18ebf_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,910 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114702f446e96cd402180a6ae05bc6eda52_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114702f446e96cd402180a6ae05bc6eda52_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,911 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b3b7284d92446f7ad4c0903108d694f_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147b3b7284d92446f7ad4c0903108d694f_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,912 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a204d7cca7494aa98a75483e4656ad09_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a204d7cca7494aa98a75483e4656ad09_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,912 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a5dd8f4095b44dfc9a3dc4c08f4e3fdb_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a5dd8f4095b44dfc9a3dc4c08f4e3fdb_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,913 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af1cf624cc50417cad876152d892b611_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114af1cf624cc50417cad876152d892b611_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,914 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ba140c790b7d4ef49bf5c2aa46fab879_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ba140c790b7d4ef49bf5c2aa46fab879_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,915 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e169d2d6b8c14bbfa7aee68359c662c1_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e169d2d6b8c14bbfa7aee68359c662c1_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,915 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f66c224e5d694839965fd1414f2023a6_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f66c224e5d694839965fd1414f2023a6_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,916 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f7f110cd69724543b9f4798dc79355ce_29dbb160edffb51f36ebfbb295b932e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f7f110cd69724543b9f4798dc79355ce_29dbb160edffb51f36ebfbb295b932e8 2024-11-14T09:45:30,916 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:45:30,918 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=103, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:30,920 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:45:30,922 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:45:30,923 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=103, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:30,923 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:45:30,923 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577530923"}]},"ts":"9223372036854775807"} 2024-11-14T09:45:30,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:45:30,926 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 29dbb160edffb51f36ebfbb295b932e8, NAME => 'TestAcidGuarantees,,1731577502560.29dbb160edffb51f36ebfbb295b932e8.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:45:30,926 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:45:30,926 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577530926"}]},"ts":"9223372036854775807"} 2024-11-14T09:45:30,928 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:45:30,929 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=103, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:30,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-14T09:45:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:45:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:45:30,981 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:30,989 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 248) - Thread LEAK? -, OpenFileDescriptor=461 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=323 (was 313) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3797 (was 3837) 2024-11-14T09:45:30,998 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=3797 2024-11-14T09:45:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:45:30,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:45:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:45:31,000 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:45:31,001 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:31,001 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 104 2024-11-14T09:45:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-14T09:45:31,001 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:45:31,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742214_1390 (size=1061) 2024-11-14T09:45:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-14T09:45:31,263 WARN [regionserver/defc576eb6b7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-14T09:45:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-14T09:45:31,412 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:45:31,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742215_1391 (size=53) 2024-11-14T09:45:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-14T09:45:31,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:45:31,824 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9542cc54aea5ce4e3db1d4ac48e4216a, disabling compactions & flushes 2024-11-14T09:45:31,824 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:31,824 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:31,824 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. after waiting 0 ms 2024-11-14T09:45:31,824 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:31,824 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:31,824 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9542cc54aea5ce4e3db1d4ac48e4216a: Waiting for close lock at 1731577531824Disabling compacts and flushes for region at 1731577531824Disabling writes for close at 1731577531824Writing region close event to WAL at 1731577531824Closed at 1731577531824 2024-11-14T09:45:31,827 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:45:31,828 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577531828"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577531828"}]},"ts":"1731577531828"} 2024-11-14T09:45:31,830 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:45:31,831 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:45:31,831 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577531831"}]},"ts":"1731577531831"} 2024-11-14T09:45:31,833 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:45:31,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, ASSIGN}] 2024-11-14T09:45:31,834 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, ASSIGN 2024-11-14T09:45:31,835 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, ASSIGN; state=OFFLINE, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=false 2024-11-14T09:45:31,986 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=9542cc54aea5ce4e3db1d4ac48e4216a, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:31,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, ASSIGN because future has completed 2024-11-14T09:45:31,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:45:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-14T09:45:32,153 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:32,153 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:45:32,155 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,155 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:45:32,155 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,156 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,158 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,159 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:32,160 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9542cc54aea5ce4e3db1d4ac48e4216a columnFamilyName A 2024-11-14T09:45:32,160 DEBUG [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:32,160 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.HStore(327): Store=9542cc54aea5ce4e3db1d4ac48e4216a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:32,160 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,161 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:32,162 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9542cc54aea5ce4e3db1d4ac48e4216a columnFamilyName B 2024-11-14T09:45:32,162 DEBUG [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:32,162 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.HStore(327): Store=9542cc54aea5ce4e3db1d4ac48e4216a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:32,162 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,163 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:45:32,163 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9542cc54aea5ce4e3db1d4ac48e4216a columnFamilyName C 2024-11-14T09:45:32,164 DEBUG [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:45:32,164 INFO [StoreOpener-9542cc54aea5ce4e3db1d4ac48e4216a-1 {}] regionserver.HStore(327): Store=9542cc54aea5ce4e3db1d4ac48e4216a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:45:32,164 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:32,164 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,165 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,165 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,167 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,167 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,167 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:45:32,169 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,171 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:45:32,172 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 9542cc54aea5ce4e3db1d4ac48e4216a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64357592, jitterRate=-0.04099714756011963}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:45:32,172 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:32,173 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 9542cc54aea5ce4e3db1d4ac48e4216a: Running coprocessor pre-open hook at 1731577532156Writing region info on filesystem at 1731577532156Initializing all the Stores at 1731577532158 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577532158Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577532158Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577532158Cleaning up temporary data from old regions at 1731577532167 (+9 ms)Running coprocessor post-open hooks at 1731577532172 (+5 ms)Region opened successfully at 1731577532173 (+1 ms) 2024-11-14T09:45:32,174 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., pid=106, masterSystemTime=1731577532146 2024-11-14T09:45:32,176 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:32,176 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:32,177 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=9542cc54aea5ce4e3db1d4ac48e4216a, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:32,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:45:32,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T09:45:32,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; OpenRegionProcedure 9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 in 188 msec 2024-11-14T09:45:32,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-14T09:45:32,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, ASSIGN in 349 msec 2024-11-14T09:45:32,185 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:45:32,185 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577532185"}]},"ts":"1731577532185"} 2024-11-14T09:45:32,187 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:45:32,187 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:45:32,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-11-14T09:45:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-14T09:45:33,152 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:33,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5aec27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684e35cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,163 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,166 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:45:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,33347,1731577407591 2024-11-14T09:45:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38ba0acf 2024-11-14T09:45:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:45:33,168 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:45:33,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,169 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f351aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a78a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,172 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745f90ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,173 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412ae47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,175 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84b4def, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,177 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53909f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,179 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd8009a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,181 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e37588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,183 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b4b191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,185 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb06446, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed4259a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784e3679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,191 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3b5a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ecb0b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,194 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc5228e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,195 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4b660c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,197 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f30374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,198 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0cd8c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb1bf9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:45:33,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:45:33,199 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:45:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:45:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:45:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5658511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:45:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:45:33,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:33,201 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:45:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1e3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:45:33,201 DEBUG [Thread-1794 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,202 DEBUG [Thread-1783 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,202 DEBUG [Thread-1789 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,202 DEBUG [Thread-1781 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,203 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,203 DEBUG [Thread-1796 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,203 DEBUG [Thread-1800 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,206 DEBUG [Thread-1798 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,208 DEBUG [Thread-1785 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,209 DEBUG [Thread-1792 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,209 DEBUG [Thread-1787 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:45:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:45:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:45:33,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:45:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2] 2024-11-14T09:45:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-14T09:45:33,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:45:33,266 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:33,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:33,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:33,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:45:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:33,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:33,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:33,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:33,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:33,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:33,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:33,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/5be39f7461d047e48f71b328daef1439 is 50, key is test_row_0/A:col10/1731577533269/Put/seqid=0 2024-11-14T09:45:33,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742216_1392 (size=16681) 2024-11-14T09:45:33,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577593297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577593299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577593300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577593301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577593306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:45:33,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577593410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:45:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:33,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577593420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577593420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577593420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577593420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:45:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:33,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:45:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577593620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577593640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577593641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577593641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577593641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/5be39f7461d047e48f71b328daef1439 2024-11-14T09:45:33,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/f657e256efc24004bbbfb63292d37e58 is 50, key is test_row_0/B:col10/1731577533269/Put/seqid=0 2024-11-14T09:45:33,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742217_1393 (size=12001) 2024-11-14T09:45:33,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/f657e256efc24004bbbfb63292d37e58 2024-11-14T09:45:33,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:45:33,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:33,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:33,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/e9b1d975911f48d4800902093bd7f2e9 is 50, key is test_row_0/C:col10/1731577533269/Put/seqid=0 2024-11-14T09:45:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742218_1394 (size=12001) 2024-11-14T09:45:33,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/e9b1d975911f48d4800902093bd7f2e9 2024-11-14T09:45:33,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/5be39f7461d047e48f71b328daef1439 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/5be39f7461d047e48f71b328daef1439 2024-11-14T09:45:33,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/5be39f7461d047e48f71b328daef1439, entries=250, sequenceid=13, filesize=16.3 K 2024-11-14T09:45:33,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/f657e256efc24004bbbfb63292d37e58 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f657e256efc24004bbbfb63292d37e58 2024-11-14T09:45:33,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f657e256efc24004bbbfb63292d37e58, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T09:45:33,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/e9b1d975911f48d4800902093bd7f2e9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e9b1d975911f48d4800902093bd7f2e9 2024-11-14T09:45:33,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e9b1d975911f48d4800902093bd7f2e9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T09:45:33,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9542cc54aea5ce4e3db1d4ac48e4216a in 497ms, sequenceid=13, compaction requested=false 2024-11-14T09:45:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:45:33,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:33,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T09:45:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:33,880 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:45:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:33,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/8fd372071b4d4260a8af321360444d85 is 50, key is test_row_0/A:col10/1731577533298/Put/seqid=0 2024-11-14T09:45:33,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742219_1395 (size=12001) 2024-11-14T09:45:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:45:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:33,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577593955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577593956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577593957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577593958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577593959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577594070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577594070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577594070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577594070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577594070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577594280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577594280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577594280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,287 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/8fd372071b4d4260a8af321360444d85 2024-11-14T09:45:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577594280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577594280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/2a72cf921cf74adeaa59ce005d71b30c is 50, key is test_row_0/B:col10/1731577533298/Put/seqid=0 2024-11-14T09:45:34,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742220_1396 (size=12001) 2024-11-14T09:45:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:45:34,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577594590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577594590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577594590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577594600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577594600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:34,696 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/2a72cf921cf74adeaa59ce005d71b30c 2024-11-14T09:45:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/4b15a917cb6443449128fa04de080e5d is 50, key is test_row_0/C:col10/1731577533298/Put/seqid=0 2024-11-14T09:45:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742221_1397 (size=12001) 2024-11-14T09:45:35,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577595100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,106 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/4b15a917cb6443449128fa04de080e5d 2024-11-14T09:45:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577595100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/8fd372071b4d4260a8af321360444d85 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/8fd372071b4d4260a8af321360444d85 2024-11-14T09:45:35,112 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/8fd372071b4d4260a8af321360444d85, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:45:35,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/2a72cf921cf74adeaa59ce005d71b30c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2a72cf921cf74adeaa59ce005d71b30c 2024-11-14T09:45:35,116 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2a72cf921cf74adeaa59ce005d71b30c, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:45:35,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577595110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/4b15a917cb6443449128fa04de080e5d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4b15a917cb6443449128fa04de080e5d 2024-11-14T09:45:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,121 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4b15a917cb6443449128fa04de080e5d, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T09:45:35,121 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1241ms, sequenceid=33, compaction requested=false 2024-11-14T09:45:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-14T09:45:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-14T09:45:35,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T09:45:35,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-14T09:45:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:35,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:45:35,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:35,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:35,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:35,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:35,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:35,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:35,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.9220 sec 2024-11-14T09:45:35,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/9fac04691cdd44fa93dab0fe64b65597 is 50, key is test_row_0/A:col10/1731577533958/Put/seqid=0 2024-11-14T09:45:35,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742222_1398 (size=14341) 2024-11-14T09:45:35,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577595184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577595193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577595300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577595300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T09:45:35,410 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:35,411 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-14T09:45:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:45:35,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:35,413 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:35,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577595510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:45:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577595520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/9fac04691cdd44fa93dab0fe64b65597 2024-11-14T09:45:35,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/7fd04601491f4a2d80968a01911e1e37 is 50, key is test_row_0/B:col10/1731577533958/Put/seqid=0 2024-11-14T09:45:35,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742223_1399 (size=12001) 2024-11-14T09:45:35,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:35,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:35,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:35,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:35,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:35,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:35,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:35,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:35,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:45:35,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577595820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577595840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:35,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:35,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:35,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:35,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:35,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:35,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/7fd04601491f4a2d80968a01911e1e37 2024-11-14T09:45:35,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/7555f9867d4e4216a79e849389636426 is 50, key is test_row_0/C:col10/1731577533958/Put/seqid=0 2024-11-14T09:45:35,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742224_1400 (size=12001) 2024-11-14T09:45:36,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:45:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577596120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577596120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:36,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:36,123 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577596120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:36,124 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:36,124 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:36,138 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:45:36,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:36,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577596340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:36,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/7555f9867d4e4216a79e849389636426 2024-11-14T09:45:36,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/9fac04691cdd44fa93dab0fe64b65597 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/9fac04691cdd44fa93dab0fe64b65597 2024-11-14T09:45:36,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/9fac04691cdd44fa93dab0fe64b65597, entries=200, sequenceid=46, filesize=14.0 K 2024-11-14T09:45:36,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/7fd04601491f4a2d80968a01911e1e37 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/7fd04601491f4a2d80968a01911e1e37 2024-11-14T09:45:36,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/7fd04601491f4a2d80968a01911e1e37, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T09:45:36,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/7555f9867d4e4216a79e849389636426 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7555f9867d4e4216a79e849389636426 2024-11-14T09:45:36,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7555f9867d4e4216a79e849389636426, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T09:45:36,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1239ms, sequenceid=46, compaction requested=true 2024-11-14T09:45:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:36,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:36,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:36,365 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:36,365 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:36,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:36,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:36,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:36,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:36,365 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:36,365 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/B is initiating minor compaction (all files) 2024-11-14T09:45:36,366 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/B in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,366 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f657e256efc24004bbbfb63292d37e58, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2a72cf921cf74adeaa59ce005d71b30c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/7fd04601491f4a2d80968a01911e1e37] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=35.2 K 2024-11-14T09:45:36,366 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:36,366 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/A is initiating minor compaction (all files) 2024-11-14T09:45:36,366 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/A in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,366 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/5be39f7461d047e48f71b328daef1439, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/8fd372071b4d4260a8af321360444d85, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/9fac04691cdd44fa93dab0fe64b65597] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=42.0 K 2024-11-14T09:45:36,366 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be39f7461d047e48f71b328daef1439, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577533265 2024-11-14T09:45:36,367 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f657e256efc24004bbbfb63292d37e58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577533267 2024-11-14T09:45:36,367 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fd372071b4d4260a8af321360444d85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731577533298 2024-11-14T09:45:36,367 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a72cf921cf74adeaa59ce005d71b30c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731577533298 2024-11-14T09:45:36,367 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fac04691cdd44fa93dab0fe64b65597, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577533945 2024-11-14T09:45:36,367 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd04601491f4a2d80968a01911e1e37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577533957 2024-11-14T09:45:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:36,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:45:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:36,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/4e1bdd6cd6574872bae942a6c0a2e489 is 50, key is test_row_0/A:col10/1731577535182/Put/seqid=0 2024-11-14T09:45:36,375 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:36,375 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#A#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:36,376 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/0fe2c609e88b4f12b2a5a2799725dcda is 50, key is test_row_0/B:col10/1731577533958/Put/seqid=0 2024-11-14T09:45:36,376 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/3cfdaaf3bcea4e35bb8787658d41f5bc is 50, key is test_row_0/A:col10/1731577533958/Put/seqid=0 2024-11-14T09:45:36,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742225_1401 (size=14341) 2024-11-14T09:45:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742227_1403 (size=12104) 2024-11-14T09:45:36,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742226_1402 (size=12104) 2024-11-14T09:45:36,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577596423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:36,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577596540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:45:36,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577596750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:36,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/4e1bdd6cd6574872bae942a6c0a2e489 2024-11-14T09:45:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/82e9269e9648466f9c0a61a41ad340b8 is 50, key is test_row_0/B:col10/1731577535182/Put/seqid=0 2024-11-14T09:45:36,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742228_1404 (size=12001) 2024-11-14T09:45:36,799 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/0fe2c609e88b4f12b2a5a2799725dcda as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/0fe2c609e88b4f12b2a5a2799725dcda 2024-11-14T09:45:36,802 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/3cfdaaf3bcea4e35bb8787658d41f5bc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/3cfdaaf3bcea4e35bb8787658d41f5bc 2024-11-14T09:45:36,803 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/B of 9542cc54aea5ce4e3db1d4ac48e4216a into 0fe2c609e88b4f12b2a5a2799725dcda(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:36,803 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:36,803 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/B, priority=13, startTime=1731577536365; duration=0sec 2024-11-14T09:45:36,803 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:36,803 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:B 2024-11-14T09:45:36,803 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:36,804 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:36,804 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/C is initiating minor compaction (all files) 2024-11-14T09:45:36,804 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/C in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,804 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e9b1d975911f48d4800902093bd7f2e9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4b15a917cb6443449128fa04de080e5d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7555f9867d4e4216a79e849389636426] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=35.2 K 2024-11-14T09:45:36,805 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b1d975911f48d4800902093bd7f2e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577533267 2024-11-14T09:45:36,805 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b15a917cb6443449128fa04de080e5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731577533298 2024-11-14T09:45:36,805 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/A of 9542cc54aea5ce4e3db1d4ac48e4216a into 3cfdaaf3bcea4e35bb8787658d41f5bc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:36,805 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 7555f9867d4e4216a79e849389636426, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577533957 2024-11-14T09:45:36,805 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:36,805 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/A, priority=13, startTime=1731577536364; duration=0sec 2024-11-14T09:45:36,805 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:36,805 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:A 2024-11-14T09:45:36,811 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#C#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:36,811 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a7bbeb28a8994d69bec904e001c7f454 is 50, key is test_row_0/C:col10/1731577533958/Put/seqid=0 2024-11-14T09:45:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742229_1405 (size=12104) 2024-11-14T09:45:36,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:36,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577597060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:37,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/82e9269e9648466f9c0a61a41ad340b8 2024-11-14T09:45:37,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/1094f6ecf9b5486bb23878dfcf096171 is 50, key is test_row_0/C:col10/1731577535182/Put/seqid=0 2024-11-14T09:45:37,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742230_1406 (size=12001) 2024-11-14T09:45:37,218 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a7bbeb28a8994d69bec904e001c7f454 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a7bbeb28a8994d69bec904e001c7f454 2024-11-14T09:45:37,221 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/C of 9542cc54aea5ce4e3db1d4ac48e4216a into a7bbeb28a8994d69bec904e001c7f454(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:37,221 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:37,222 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/C, priority=13, startTime=1731577536365; duration=0sec 2024-11-14T09:45:37,222 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:37,222 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:C 2024-11-14T09:45:37,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:37,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577597360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:37,364 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:37,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:37,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:37,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:45:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577597570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:37,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/1094f6ecf9b5486bb23878dfcf096171 2024-11-14T09:45:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/4e1bdd6cd6574872bae942a6c0a2e489 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/4e1bdd6cd6574872bae942a6c0a2e489 2024-11-14T09:45:37,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/4e1bdd6cd6574872bae942a6c0a2e489, entries=200, sequenceid=66, filesize=14.0 K 2024-11-14T09:45:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/82e9269e9648466f9c0a61a41ad340b8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/82e9269e9648466f9c0a61a41ad340b8 2024-11-14T09:45:37,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/82e9269e9648466f9c0a61a41ad340b8, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T09:45:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/1094f6ecf9b5486bb23878dfcf096171 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1094f6ecf9b5486bb23878dfcf096171 2024-11-14T09:45:37,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1094f6ecf9b5486bb23878dfcf096171, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T09:45:37,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1253ms, sequenceid=66, compaction requested=false 2024-11-14T09:45:37,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:37,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T09:45:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:37,701 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:45:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:37,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/17a8dc9078ff4fc68cf38fa937b0d1eb is 50, key is test_row_0/A:col10/1731577536421/Put/seqid=0 2024-11-14T09:45:37,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742231_1407 (size=12001) 2024-11-14T09:45:38,112 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/17a8dc9078ff4fc68cf38fa937b0d1eb 2024-11-14T09:45:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/b4d21444452048508441a81f98a2eae2 is 50, key is test_row_0/B:col10/1731577536421/Put/seqid=0 2024-11-14T09:45:38,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742232_1408 (size=12001) 2024-11-14T09:45:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:38,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:38,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577598180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577598189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577598190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577598300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577598300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577598310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577598510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577598510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,522 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/b4d21444452048508441a81f98a2eae2 2024-11-14T09:45:38,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577598520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/f9e5bf760de74820864993d3d58b7bb8 is 50, key is test_row_0/C:col10/1731577536421/Put/seqid=0 2024-11-14T09:45:38,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742233_1409 (size=12001) 2024-11-14T09:45:38,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577598580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,586 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577598820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577598820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577598830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:38,932 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/f9e5bf760de74820864993d3d58b7bb8 2024-11-14T09:45:38,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/17a8dc9078ff4fc68cf38fa937b0d1eb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/17a8dc9078ff4fc68cf38fa937b0d1eb 2024-11-14T09:45:38,943 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/17a8dc9078ff4fc68cf38fa937b0d1eb, entries=150, sequenceid=81, filesize=11.7 K 2024-11-14T09:45:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/b4d21444452048508441a81f98a2eae2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/b4d21444452048508441a81f98a2eae2 2024-11-14T09:45:38,946 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/b4d21444452048508441a81f98a2eae2, entries=150, sequenceid=81, filesize=11.7 K 2024-11-14T09:45:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/f9e5bf760de74820864993d3d58b7bb8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f9e5bf760de74820864993d3d58b7bb8 2024-11-14T09:45:38,951 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f9e5bf760de74820864993d3d58b7bb8, entries=150, sequenceid=81, filesize=11.7 K 2024-11-14T09:45:38,952 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1251ms, sequenceid=81, compaction requested=true 2024-11-14T09:45:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-14T09:45:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-14T09:45:38,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T09:45:38,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5410 sec 2024-11-14T09:45:38,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.5450 sec 2024-11-14T09:45:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:39,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T09:45:39,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:39,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:39,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:39,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:39,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:39,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:39,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/62fac6776bce49fc9c2756344c13a35c is 50, key is test_row_0/A:col10/1731577538178/Put/seqid=0 2024-11-14T09:45:39,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742234_1410 (size=11997) 2024-11-14T09:45:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577599359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577599360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577599361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577599390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,396 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577599470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577599471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577599471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:45:39,581 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:39,581 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-14T09:45:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:39,583 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:39,583 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:39,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577599691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577599691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577599691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:39,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:39,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:39,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:39,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:39,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:39,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/62fac6776bce49fc9c2756344c13a35c 2024-11-14T09:45:39,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/e3ee48d39ca14491a6aec4cee1353088 is 50, key is test_row_0/B:col10/1731577538178/Put/seqid=0 2024-11-14T09:45:39,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742235_1411 (size=9657) 2024-11-14T09:45:39,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:39,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:40,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577600010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577600010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577600010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/e3ee48d39ca14491a6aec4cee1353088 2024-11-14T09:45:40,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/e0f4cf4374d44e89b77041c8ed9953c5 is 50, key is test_row_0/C:col10/1731577538178/Put/seqid=0 2024-11-14T09:45:40,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742236_1412 (size=9657) 2024-11-14T09:45:40,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:40,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:40,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:40,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577600530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577600530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577600531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/e0f4cf4374d44e89b77041c8ed9953c5 2024-11-14T09:45:40,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/62fac6776bce49fc9c2756344c13a35c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/62fac6776bce49fc9c2756344c13a35c 2024-11-14T09:45:40,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/62fac6776bce49fc9c2756344c13a35c, entries=150, sequenceid=102, filesize=11.7 K 2024-11-14T09:45:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/e3ee48d39ca14491a6aec4cee1353088 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/e3ee48d39ca14491a6aec4cee1353088 2024-11-14T09:45:40,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/e3ee48d39ca14491a6aec4cee1353088, entries=100, sequenceid=102, filesize=9.4 K 2024-11-14T09:45:40,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/e0f4cf4374d44e89b77041c8ed9953c5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e0f4cf4374d44e89b77041c8ed9953c5 2024-11-14T09:45:40,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e0f4cf4374d44e89b77041c8ed9953c5, entries=100, sequenceid=102, filesize=9.4 K 2024-11-14T09:45:40,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1252ms, sequenceid=102, compaction requested=true 2024-11-14T09:45:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:40,599 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:40,599 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:40,600 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50443 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:40,600 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/A is initiating minor compaction (all files) 2024-11-14T09:45:40,600 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/A in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,600 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/3cfdaaf3bcea4e35bb8787658d41f5bc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/4e1bdd6cd6574872bae942a6c0a2e489, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/17a8dc9078ff4fc68cf38fa937b0d1eb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/62fac6776bce49fc9c2756344c13a35c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=49.3 K 2024-11-14T09:45:40,600 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:40,600 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/B is initiating minor compaction (all files) 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cfdaaf3bcea4e35bb8787658d41f5bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577533957 2024-11-14T09:45:40,601 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/B in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,601 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/0fe2c609e88b4f12b2a5a2799725dcda, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/82e9269e9648466f9c0a61a41ad340b8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/b4d21444452048508441a81f98a2eae2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/e3ee48d39ca14491a6aec4cee1353088] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=44.7 K 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e1bdd6cd6574872bae942a6c0a2e489, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577535175 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe2c609e88b4f12b2a5a2799725dcda, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577533957 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a8dc9078ff4fc68cf38fa937b0d1eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577536406 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e9269e9648466f9c0a61a41ad340b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577535175 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62fac6776bce49fc9c2756344c13a35c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731577538178 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d21444452048508441a81f98a2eae2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577536406 2024-11-14T09:45:40,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e3ee48d39ca14491a6aec4cee1353088, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731577538178 2024-11-14T09:45:40,607 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#A#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:40,608 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:40,608 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/10c67b79d7ac4c649d942b817d436757 is 50, key is test_row_0/A:col10/1731577538178/Put/seqid=0 2024-11-14T09:45:40,608 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/3ca2f6e30c0f4190830c893e863af9ff is 50, key is test_row_0/B:col10/1731577538178/Put/seqid=0 2024-11-14T09:45:40,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:45:40,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:40,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:40,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:40,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:40,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:40,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:40,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d56973bcade94010aec7c182c92b7e27 is 50, key is test_row_0/A:col10/1731577539357/Put/seqid=0 2024-11-14T09:45:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742238_1414 (size=12241) 2024-11-14T09:45:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742237_1413 (size=12241) 2024-11-14T09:45:40,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742239_1415 (size=14341) 2024-11-14T09:45:40,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:40,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:40,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577600763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:40,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:40,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577600881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:40,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:40,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,027 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/10c67b79d7ac4c649d942b817d436757 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/10c67b79d7ac4c649d942b817d436757 2024-11-14T09:45:41,028 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/3ca2f6e30c0f4190830c893e863af9ff as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3ca2f6e30c0f4190830c893e863af9ff 2024-11-14T09:45:41,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d56973bcade94010aec7c182c92b7e27 2024-11-14T09:45:41,031 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/B of 9542cc54aea5ce4e3db1d4ac48e4216a into 3ca2f6e30c0f4190830c893e863af9ff(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:41,031 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:41,031 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/B, priority=12, startTime=1731577540599; duration=0sec 2024-11-14T09:45:41,031 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:41,031 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:B 2024-11-14T09:45:41,031 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:41,032 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/A of 9542cc54aea5ce4e3db1d4ac48e4216a into 10c67b79d7ac4c649d942b817d436757(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:41,032 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:41,032 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/A, priority=12, startTime=1731577540599; duration=0sec 2024-11-14T09:45:41,032 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:41,032 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:A 2024-11-14T09:45:41,033 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:41,033 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/C is initiating minor compaction (all files) 2024-11-14T09:45:41,033 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/C in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,033 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a7bbeb28a8994d69bec904e001c7f454, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1094f6ecf9b5486bb23878dfcf096171, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f9e5bf760de74820864993d3d58b7bb8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e0f4cf4374d44e89b77041c8ed9953c5] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=44.7 K 2024-11-14T09:45:41,033 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a7bbeb28a8994d69bec904e001c7f454, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731577533957 2024-11-14T09:45:41,034 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 1094f6ecf9b5486bb23878dfcf096171, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731577535175 2024-11-14T09:45:41,034 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e5bf760de74820864993d3d58b7bb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577536406 2024-11-14T09:45:41,034 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e0f4cf4374d44e89b77041c8ed9953c5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731577538178 2024-11-14T09:45:41,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/f4b9190ba1c04c50b3622a32576f0710 is 50, key is test_row_0/B:col10/1731577539357/Put/seqid=0 2024-11-14T09:45:41,042 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:41,043 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/1b48c3e6f0e94e738965cf4d79da2a70 is 50, key is test_row_0/C:col10/1731577538178/Put/seqid=0 2024-11-14T09:45:41,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742240_1416 (size=12001) 2024-11-14T09:45:41,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742241_1417 (size=12241) 2024-11-14T09:45:41,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:41,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577601100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:41,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577601420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:41,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/f4b9190ba1c04c50b3622a32576f0710 2024-11-14T09:45:41,452 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/1b48c3e6f0e94e738965cf4d79da2a70 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1b48c3e6f0e94e738965cf4d79da2a70 2024-11-14T09:45:41,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/0e5456d26c98444baf810071be08230a is 50, key is test_row_0/C:col10/1731577539357/Put/seqid=0 2024-11-14T09:45:41,456 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/C of 9542cc54aea5ce4e3db1d4ac48e4216a into 1b48c3e6f0e94e738965cf4d79da2a70(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:41,456 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:41,456 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/C, priority=12, startTime=1731577540599; duration=0sec 2024-11-14T09:45:41,456 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:41,456 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:C 2024-11-14T09:45:41,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742242_1418 (size=12001) 2024-11-14T09:45:41,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577601540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:41,547 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577601550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577601551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:41,557 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:41,557 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:41,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:41,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:41,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:41,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/0e5456d26c98444baf810071be08230a 2024-11-14T09:45:41,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d56973bcade94010aec7c182c92b7e27 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d56973bcade94010aec7c182c92b7e27 2024-11-14T09:45:41,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d56973bcade94010aec7c182c92b7e27, entries=200, sequenceid=114, filesize=14.0 K 2024-11-14T09:45:41,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/f4b9190ba1c04c50b3622a32576f0710 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f4b9190ba1c04c50b3622a32576f0710 2024-11-14T09:45:41,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f4b9190ba1c04c50b3622a32576f0710, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T09:45:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:41,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/0e5456d26c98444baf810071be08230a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/0e5456d26c98444baf810071be08230a 2024-11-14T09:45:41,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/0e5456d26c98444baf810071be08230a, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T09:45:41,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1270ms, sequenceid=114, compaction requested=false 2024-11-14T09:45:41,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:41,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577540727, max timestamp=1731577541934Num uniques 859; 2024-11-14T09:45:41,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577540727, max timestamp=1731577541934Num uniques 859; 2024-11-14T09:45:41,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:41,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577540727, max timestamp=1731577541934Num uniques 859; 2024-11-14T09:45:41,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:41,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:41,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-14T09:45:41,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:41,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:41,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:41,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:41,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:41,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:41,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/f0a36db5728641d68f323a220e1902bd is 50, key is test_row_0/A:col10/1731577540762/Put/seqid=0 2024-11-14T09:45:41,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742243_1419 (size=14541) 2024-11-14T09:45:42,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:42,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:42,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577542023, max timestamp=1731577542049Num uniques 859; 2024-11-14T09:45:42,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577542023, max timestamp=1731577542049Num uniques 859; 2024-11-14T09:45:42,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577542023, max timestamp=1731577542049Num uniques 859; 2024-11-14T09:45:42,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:42,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:42,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:45:42,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:42,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:42,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731577542118, max timestamp=1731577542175Num uniques 1018; 2024-11-14T09:45:42,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731577542118, max timestamp=1731577542175Num uniques 1018; 2024-11-14T09:45:42,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731577542118, max timestamp=1731577542175Num uniques 1018; 2024-11-14T09:45:42,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:45:42,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:45:42,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T09:45:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731577542262, max timestamp=1731577542298Num uniques 1027; 2024-11-14T09:45:42,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T09:45:42,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731577542262, max timestamp=1731577542298Num uniques 1027; 2024-11-14T09:45:42,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731577542262, max timestamp=1731577542298Num uniques 1027; 2024-11-14T09:45:42,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T09:45:42,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T09:45:42,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:42,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:42,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/f0a36db5728641d68f323a220e1902bd 2024-11-14T09:45:42,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/96fc2decea8a453ea5fa38ef138617c7 is 50, key is test_row_0/B:col10/1731577540762/Put/seqid=0 2024-11-14T09:45:42,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742244_1420 (size=12151) 2024-11-14T09:45:42,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731577542360, max timestamp=1731577542417Num uniques 1036; 2024-11-14T09:45:42,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T09:45:42,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731577542360, max timestamp=1731577542417Num uniques 1036; 2024-11-14T09:45:42,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731577542360, max timestamp=1731577542417Num uniques 1036; 2024-11-14T09:45:42,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T09:45:42,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T09:45:42,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:42,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731577542496, max timestamp=1731577542553Num uniques 1045; 2024-11-14T09:45:42,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731577542496, max timestamp=1731577542553Num uniques 1045; 2024-11-14T09:45:42,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731577542496, max timestamp=1731577542553Num uniques 1045; 2024-11-14T09:45:42,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T09:45:42,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T09:45:42,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T09:45:42,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731577542631, max timestamp=1731577542676Num uniques 1054; 2024-11-14T09:45:42,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731577542631, max timestamp=1731577542676Num uniques 1054; 2024-11-14T09:45:42,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T09:45:42,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T09:45:42,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731577542631, max timestamp=1731577542676Num uniques 1054; 2024-11-14T09:45:42,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T09:45:42,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/96fc2decea8a453ea5fa38ef138617c7 2024-11-14T09:45:42,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/47c15279a3b640958918893580ed7bdd is 50, key is test_row_0/C:col10/1731577540762/Put/seqid=0 2024-11-14T09:45:42,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742245_1421 (size=12151) 2024-11-14T09:45:42,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:42,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:42,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731577542798, max timestamp=1731577542829Num uniques 1013; 2024-11-14T09:45:42,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T09:45:42,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731577542798, max timestamp=1731577542829Num uniques 1013; 2024-11-14T09:45:42,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T09:45:42,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731577542798, max timestamp=1731577542829Num uniques 1013; 2024-11-14T09:45:42,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T09:45:42,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:42,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:42,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:42,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731577542949, max timestamp=1731577542984Num uniques 1022; 2024-11-14T09:45:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731577542949, max timestamp=1731577542984Num uniques 1022; 2024-11-14T09:45:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731577542949, max timestamp=1731577542984Num uniques 1022; 2024-11-14T09:45:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-14T09:45:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-14T09:45:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-14T09:45:43,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:43,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731577543073, max timestamp=1731577543115Num uniques 1031; 2024-11-14T09:45:43,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731577543073, max timestamp=1731577543115Num uniques 1031; 2024-11-14T09:45:43,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731577543073, max timestamp=1731577543115Num uniques 1031; 2024-11-14T09:45:43,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T09:45:43,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T09:45:43,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T09:45:43,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/47c15279a3b640958918893580ed7bdd 2024-11-14T09:45:43,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/f0a36db5728641d68f323a220e1902bd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/f0a36db5728641d68f323a220e1902bd 2024-11-14T09:45:43,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/f0a36db5728641d68f323a220e1902bd, entries=200, sequenceid=138, filesize=14.2 K 2024-11-14T09:45:43,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/96fc2decea8a453ea5fa38ef138617c7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/96fc2decea8a453ea5fa38ef138617c7 2024-11-14T09:45:43,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/96fc2decea8a453ea5fa38ef138617c7, entries=150, sequenceid=138, filesize=11.9 K 2024-11-14T09:45:43,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/47c15279a3b640958918893580ed7bdd as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/47c15279a3b640958918893580ed7bdd 2024-11-14T09:45:43,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/47c15279a3b640958918893580ed7bdd, entries=150, sequenceid=138, filesize=11.9 K 2024-11-14T09:45:43,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=87.22 KB/89310 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1242ms, sequenceid=138, compaction requested=true 2024-11-14T09:45:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:43,181 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:43,181 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:43,181 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41123 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:43,181 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:43,181 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/A is initiating minor compaction (all files) 2024-11-14T09:45:43,181 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/B is initiating minor compaction (all files) 2024-11-14T09:45:43,182 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/A in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,182 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/B in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,182 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/10c67b79d7ac4c649d942b817d436757, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d56973bcade94010aec7c182c92b7e27, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/f0a36db5728641d68f323a220e1902bd] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=40.2 K 2024-11-14T09:45:43,182 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3ca2f6e30c0f4190830c893e863af9ff, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f4b9190ba1c04c50b3622a32576f0710, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/96fc2decea8a453ea5fa38ef138617c7] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=35.5 K 2024-11-14T09:45:43,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca2f6e30c0f4190830c893e863af9ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731577536406 2024-11-14T09:45:43,182 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c67b79d7ac4c649d942b817d436757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731577536406 2024-11-14T09:45:43,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b9190ba1c04c50b3622a32576f0710, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577539357 2024-11-14T09:45:43,182 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d56973bcade94010aec7c182c92b7e27, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577539357 2024-11-14T09:45:43,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fc2decea8a453ea5fa38ef138617c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731577540739 2024-11-14T09:45:43,182 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0a36db5728641d68f323a220e1902bd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731577540727 2024-11-14T09:45:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:43,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=93.93 KB heapSize=263.39 KB 2024-11-14T09:45:43,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:43,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:43,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:43,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:43,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:43,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:45:43,188 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:43,189 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/3e391c3c8da34cc583ac61e8a6efdb45 is 50, key is test_row_0/B:col10/1731577540762/Put/seqid=0 2024-11-14T09:45:43,189 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#A#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:43,189 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e0ed8c565ad342a39a4630f340abe056 is 50, key is test_row_0/A:col10/1731577540762/Put/seqid=0 2024-11-14T09:45:43,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c315313ad4b1456798e01d13957d40dc is 50, key is test_row_0/A:col10/1731577543172/Put/seqid=0 2024-11-14T09:45:43,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742247_1423 (size=12493) 2024-11-14T09:45:43,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742248_1424 (size=14741) 2024-11-14T09:45:43,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c315313ad4b1456798e01d13957d40dc 2024-11-14T09:45:43,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742246_1422 (size=12493) 2024-11-14T09:45:43,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/aa7d50a02bf94286a6d02e6c3b57324f is 50, key is test_row_0/B:col10/1731577543172/Put/seqid=0 2024-11-14T09:45:43,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742249_1425 (size=12301) 2024-11-14T09:45:43,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577603331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:43,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577603411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,419 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577603450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:43,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577603560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,569 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577603580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577603580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,591 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:43,591 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:43,603 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e0ed8c565ad342a39a4630f340abe056 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e0ed8c565ad342a39a4630f340abe056 2024-11-14T09:45:43,607 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/A of 9542cc54aea5ce4e3db1d4ac48e4216a into e0ed8c565ad342a39a4630f340abe056(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:43,607 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:43,607 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/A, priority=13, startTime=1731577543181; duration=0sec 2024-11-14T09:45:43,607 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:43,607 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:A 2024-11-14T09:45:43,607 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:45:43,608 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:45:43,608 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/C is initiating minor compaction (all files) 2024-11-14T09:45:43,608 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/C in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,608 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1b48c3e6f0e94e738965cf4d79da2a70, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/0e5456d26c98444baf810071be08230a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/47c15279a3b640958918893580ed7bdd] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=35.5 K 2024-11-14T09:45:43,608 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b48c3e6f0e94e738965cf4d79da2a70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731577536406 2024-11-14T09:45:43,608 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5456d26c98444baf810071be08230a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577539357 2024-11-14T09:45:43,609 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47c15279a3b640958918893580ed7bdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731577540739 2024-11-14T09:45:43,609 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/3e391c3c8da34cc583ac61e8a6efdb45 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3e391c3c8da34cc583ac61e8a6efdb45 2024-11-14T09:45:43,612 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/B of 9542cc54aea5ce4e3db1d4ac48e4216a into 3e391c3c8da34cc583ac61e8a6efdb45(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:43,612 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:43,612 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/B, priority=13, startTime=1731577543181; duration=0sec 2024-11-14T09:45:43,613 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:43,613 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:B 2024-11-14T09:45:43,614 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#C#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:43,615 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/615268a31fde479baab788d60ef781aa is 50, key is test_row_0/C:col10/1731577540762/Put/seqid=0 2024-11-14T09:45:43,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/aa7d50a02bf94286a6d02e6c3b57324f 2024-11-14T09:45:43,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742250_1426 (size=12493) 2024-11-14T09:45:43,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/727fd180a52b4fc5b26826d8561fa1e8 is 50, key is test_row_0/C:col10/1731577543172/Put/seqid=0 2024-11-14T09:45:43,626 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/615268a31fde479baab788d60ef781aa as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/615268a31fde479baab788d60ef781aa 2024-11-14T09:45:43,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742251_1427 (size=12301) 2024-11-14T09:45:43,630 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/C of 9542cc54aea5ce4e3db1d4ac48e4216a into 615268a31fde479baab788d60ef781aa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:43,630 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:43,630 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/C, priority=13, startTime=1731577543181; duration=0sec 2024-11-14T09:45:43,630 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:43,630 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:C 2024-11-14T09:45:43,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577603670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:43,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:43,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:43,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:43,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:43,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:43,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577603981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:44,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:44,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:44,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:44,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:44,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:44,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:44,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/727fd180a52b4fc5b26826d8561fa1e8 2024-11-14T09:45:44,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c315313ad4b1456798e01d13957d40dc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c315313ad4b1456798e01d13957d40dc 2024-11-14T09:45:44,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c315313ad4b1456798e01d13957d40dc, entries=200, sequenceid=306, filesize=14.4 K 2024-11-14T09:45:44,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/aa7d50a02bf94286a6d02e6c3b57324f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/aa7d50a02bf94286a6d02e6c3b57324f 2024-11-14T09:45:44,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/aa7d50a02bf94286a6d02e6c3b57324f, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T09:45:44,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/727fd180a52b4fc5b26826d8561fa1e8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/727fd180a52b4fc5b26826d8561fa1e8 2024-11-14T09:45:44,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/727fd180a52b4fc5b26826d8561fa1e8, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T09:45:44,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~263.34 KB/269664, currentSize=87.22 KB/89310 for 9542cc54aea5ce4e3db1d4ac48e4216a in 851ms, sequenceid=306, compaction requested=false 2024-11-14T09:45:44,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:44,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T09:45:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:44,163 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:45:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d111730aa64642b98942912fca685ed7 is 50, key is test_row_0/A:col10/1731577543292/Put/seqid=0 2024-11-14T09:45:44,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742252_1428 (size=12301) 2024-11-14T09:45:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:44,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:44,569 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d111730aa64642b98942912fca685ed7 2024-11-14T09:45:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/a089655d13524fbc888341f718473e36 is 50, key is test_row_0/B:col10/1731577543292/Put/seqid=0 2024-11-14T09:45:44,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742253_1429 (size=12301) 2024-11-14T09:45:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577604585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:44,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577604700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:44,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577604910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:44,977 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/a089655d13524fbc888341f718473e36 2024-11-14T09:45:44,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a5c2306f17a44530b7382ed4729838b1 is 50, key is test_row_0/C:col10/1731577543292/Put/seqid=0 2024-11-14T09:45:44,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742254_1430 (size=12301) 2024-11-14T09:45:45,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577605230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:45,386 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a5c2306f17a44530b7382ed4729838b1 2024-11-14T09:45:45,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d111730aa64642b98942912fca685ed7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d111730aa64642b98942912fca685ed7 2024-11-14T09:45:45,393 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d111730aa64642b98942912fca685ed7, entries=150, sequenceid=325, filesize=12.0 K 2024-11-14T09:45:45,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/a089655d13524fbc888341f718473e36 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a089655d13524fbc888341f718473e36 2024-11-14T09:45:45,396 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a089655d13524fbc888341f718473e36, entries=150, sequenceid=325, filesize=12.0 K 2024-11-14T09:45:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a5c2306f17a44530b7382ed4729838b1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a5c2306f17a44530b7382ed4729838b1 2024-11-14T09:45:45,400 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a5c2306f17a44530b7382ed4729838b1, entries=150, sequenceid=325, filesize=12.0 K 2024-11-14T09:45:45,401 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1239ms, sequenceid=325, compaction requested=true 2024-11-14T09:45:45,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:45,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:45,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-14T09:45:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-14T09:45:45,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T09:45:45,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.8190 sec 2024-11-14T09:45:45,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 5.8230 sec 2024-11-14T09:45:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:45,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:45:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:45,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:45,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/b0f72ad7e84140e7b9919ba52e486230 is 50, key is test_row_0/A:col10/1731577544564/Put/seqid=0 2024-11-14T09:45:45,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742255_1431 (size=14741) 2024-11-14T09:45:45,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577605833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:45,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577605950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:46,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/b0f72ad7e84140e7b9919ba52e486230 2024-11-14T09:45:46,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/48d84de56ee643e38c5fec0cfcaff9ec is 50, key is test_row_0/B:col10/1731577544564/Put/seqid=0 2024-11-14T09:45:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577606161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:46,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742256_1432 (size=12301) 2024-11-14T09:45:46,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577606480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:46,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/48d84de56ee643e38c5fec0cfcaff9ec 2024-11-14T09:45:46,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/29ad2a25ec024d619946c8804b721e1d is 50, key is test_row_0/C:col10/1731577544564/Put/seqid=0 2024-11-14T09:45:46,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742257_1433 (size=12301) 2024-11-14T09:45:46,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/29ad2a25ec024d619946c8804b721e1d 2024-11-14T09:45:46,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/b0f72ad7e84140e7b9919ba52e486230 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/b0f72ad7e84140e7b9919ba52e486230 2024-11-14T09:45:46,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/b0f72ad7e84140e7b9919ba52e486230, entries=200, sequenceid=342, filesize=14.4 K 2024-11-14T09:45:46,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/48d84de56ee643e38c5fec0cfcaff9ec as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/48d84de56ee643e38c5fec0cfcaff9ec 2024-11-14T09:45:46,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/48d84de56ee643e38c5fec0cfcaff9ec, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:45:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/29ad2a25ec024d619946c8804b721e1d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29ad2a25ec024d619946c8804b721e1d 2024-11-14T09:45:46,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29ad2a25ec024d619946c8804b721e1d, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:45:46,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1242ms, sequenceid=342, compaction requested=true 2024-11-14T09:45:46,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:46,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:46,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:46,994 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:46,994 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:46,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:46,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:46,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:46,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:46,996 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54276 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:46,996 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/A is initiating minor compaction (all files) 2024-11-14T09:45:46,996 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:46,996 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/B is initiating minor compaction (all files) 2024-11-14T09:45:46,996 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/A in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:46,996 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/B in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:46,996 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e0ed8c565ad342a39a4630f340abe056, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c315313ad4b1456798e01d13957d40dc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d111730aa64642b98942912fca685ed7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/b0f72ad7e84140e7b9919ba52e486230] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=53.0 K 2024-11-14T09:45:46,996 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3e391c3c8da34cc583ac61e8a6efdb45, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/aa7d50a02bf94286a6d02e6c3b57324f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a089655d13524fbc888341f718473e36, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/48d84de56ee643e38c5fec0cfcaff9ec] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=48.2 K 2024-11-14T09:45:46,996 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0ed8c565ad342a39a4630f340abe056, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731577540739 2024-11-14T09:45:46,996 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e391c3c8da34cc583ac61e8a6efdb45, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731577540739 2024-11-14T09:45:46,997 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c315313ad4b1456798e01d13957d40dc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731577543131 2024-11-14T09:45:46,997 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting aa7d50a02bf94286a6d02e6c3b57324f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731577543147 2024-11-14T09:45:46,997 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d111730aa64642b98942912fca685ed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731577543292 2024-11-14T09:45:46,997 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a089655d13524fbc888341f718473e36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731577543292 2024-11-14T09:45:46,997 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d84de56ee643e38c5fec0cfcaff9ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577544540 2024-11-14T09:45:46,997 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0f72ad7e84140e7b9919ba52e486230, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577544540 2024-11-14T09:45:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:47,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:45:47,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:47,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:47,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:47,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:47,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:47,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:47,010 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:47,011 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/74867f0dbe8d4ec4a9db8f170927444f is 50, key is test_row_0/A:col10/1731577544564/Put/seqid=0 2024-11-14T09:45:47,019 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:47,020 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/be103d73c0234bb18a5238aaa6ed0648 is 50, key is test_row_0/B:col10/1731577544564/Put/seqid=0 2024-11-14T09:45:47,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e6f463260a0640e3b8f1ae3e6ed19be7 is 50, key is test_row_0/A:col10/1731577547007/Put/seqid=0 2024-11-14T09:45:47,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742258_1434 (size=12779) 2024-11-14T09:45:47,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742259_1435 (size=12779) 2024-11-14T09:45:47,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742260_1436 (size=14741) 2024-11-14T09:45:47,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e6f463260a0640e3b8f1ae3e6ed19be7 2024-11-14T09:45:47,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/a72b1a3ab06b47898a2bc3a651a8d806 is 50, key is test_row_0/B:col10/1731577547007/Put/seqid=0 2024-11-14T09:45:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742261_1437 (size=12301) 2024-11-14T09:45:47,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577607119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:47,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577607230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:47,430 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/74867f0dbe8d4ec4a9db8f170927444f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/74867f0dbe8d4ec4a9db8f170927444f 2024-11-14T09:45:47,434 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/A of 9542cc54aea5ce4e3db1d4ac48e4216a into 74867f0dbe8d4ec4a9db8f170927444f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:47,434 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:47,434 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/A, priority=12, startTime=1731577546994; duration=0sec 2024-11-14T09:45:47,434 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:47,434 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:A 2024-11-14T09:45:47,434 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:47,436 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:47,436 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/C is initiating minor compaction (all files) 2024-11-14T09:45:47,436 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/C in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:47,436 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/615268a31fde479baab788d60ef781aa, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/727fd180a52b4fc5b26826d8561fa1e8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a5c2306f17a44530b7382ed4729838b1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29ad2a25ec024d619946c8804b721e1d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=48.2 K 2024-11-14T09:45:47,436 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 615268a31fde479baab788d60ef781aa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731577540739 2024-11-14T09:45:47,436 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 727fd180a52b4fc5b26826d8561fa1e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731577543147 2024-11-14T09:45:47,437 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c2306f17a44530b7382ed4729838b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731577543292 2024-11-14T09:45:47,437 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ad2a25ec024d619946c8804b721e1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577544540 2024-11-14T09:45:47,437 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/be103d73c0234bb18a5238aaa6ed0648 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/be103d73c0234bb18a5238aaa6ed0648 2024-11-14T09:45:47,441 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/B of 9542cc54aea5ce4e3db1d4ac48e4216a into be103d73c0234bb18a5238aaa6ed0648(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:47,441 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:47,441 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/B, priority=12, startTime=1731577546994; duration=0sec 2024-11-14T09:45:47,441 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:47,441 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:B 2024-11-14T09:45:47,443 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:47,444 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/85914010ec9b458692542b820328a652 is 50, key is test_row_0/C:col10/1731577544564/Put/seqid=0 2024-11-14T09:45:47,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742262_1438 (size=12779) 2024-11-14T09:45:47,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/a72b1a3ab06b47898a2bc3a651a8d806 2024-11-14T09:45:47,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/b405a4a818c34e20ace1d77a03aad412 is 50, key is test_row_0/C:col10/1731577547007/Put/seqid=0 2024-11-14T09:45:47,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577607450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:47,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742263_1439 (size=12301) 2024-11-14T09:45:47,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42734 deadline: 1731577607590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:47,596 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:47,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42722 deadline: 1731577607610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:47,619 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:47,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42794 deadline: 1731577607630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:47,638 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:47,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577607760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T09:45:47,770 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:47,770 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-14T09:45:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:45:47,772 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:47,772 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:47,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:47,850 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/85914010ec9b458692542b820328a652 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/85914010ec9b458692542b820328a652 2024-11-14T09:45:47,853 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/C of 9542cc54aea5ce4e3db1d4ac48e4216a into 85914010ec9b458692542b820328a652(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:47,853 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:47,854 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/C, priority=12, startTime=1731577546995; duration=0sec 2024-11-14T09:45:47,854 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:47,854 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:C 2024-11-14T09:45:47,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/b405a4a818c34e20ace1d77a03aad412 2024-11-14T09:45:47,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e6f463260a0640e3b8f1ae3e6ed19be7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e6f463260a0640e3b8f1ae3e6ed19be7 2024-11-14T09:45:47,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e6f463260a0640e3b8f1ae3e6ed19be7, entries=200, sequenceid=358, filesize=14.4 K 2024-11-14T09:45:47,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/a72b1a3ab06b47898a2bc3a651a8d806 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a72b1a3ab06b47898a2bc3a651a8d806 2024-11-14T09:45:47,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a72b1a3ab06b47898a2bc3a651a8d806, entries=150, sequenceid=358, filesize=12.0 K 2024-11-14T09:45:47,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/b405a4a818c34e20ace1d77a03aad412 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/b405a4a818c34e20ace1d77a03aad412 2024-11-14T09:45:47,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/b405a4a818c34e20ace1d77a03aad412, entries=150, sequenceid=358, filesize=12.0 K 2024-11-14T09:45:47,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9542cc54aea5ce4e3db1d4ac48e4216a in 865ms, sequenceid=358, compaction requested=false 2024-11-14T09:45:47,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:45:47,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T09:45:47,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:47,924 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:45:47,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:47,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c19b486d86a545158849ee4f024fd779 is 50, key is test_row_0/A:col10/1731577547118/Put/seqid=0 2024-11-14T09:45:47,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742264_1440 (size=12301) 2024-11-14T09:45:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:45:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:48,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:48,334 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c19b486d86a545158849ee4f024fd779 2024-11-14T09:45:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/49daa4fe9d354aa8b36e6f8437b21ac1 is 50, key is test_row_0/B:col10/1731577547118/Put/seqid=0 2024-11-14T09:45:48,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742265_1441 (size=12301) 2024-11-14T09:45:48,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577608370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:45:48,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577608480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577608700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:48,745 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/49daa4fe9d354aa8b36e6f8437b21ac1 2024-11-14T09:45:48,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/f772217da4734d62bcf3b935b0351531 is 50, key is test_row_0/C:col10/1731577547118/Put/seqid=0 2024-11-14T09:45:48,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742266_1442 (size=12301) 2024-11-14T09:45:48,755 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/f772217da4734d62bcf3b935b0351531 2024-11-14T09:45:48,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c19b486d86a545158849ee4f024fd779 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c19b486d86a545158849ee4f024fd779 2024-11-14T09:45:48,761 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c19b486d86a545158849ee4f024fd779, entries=150, sequenceid=377, filesize=12.0 K 2024-11-14T09:45:48,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/49daa4fe9d354aa8b36e6f8437b21ac1 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/49daa4fe9d354aa8b36e6f8437b21ac1 2024-11-14T09:45:48,764 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/49daa4fe9d354aa8b36e6f8437b21ac1, entries=150, sequenceid=377, filesize=12.0 K 2024-11-14T09:45:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/f772217da4734d62bcf3b935b0351531 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f772217da4734d62bcf3b935b0351531 2024-11-14T09:45:48,767 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f772217da4734d62bcf3b935b0351531, entries=150, sequenceid=377, filesize=12.0 K 2024-11-14T09:45:48,770 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9542cc54aea5ce4e3db1d4ac48e4216a in 846ms, sequenceid=377, compaction requested=true 2024-11-14T09:45:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-14T09:45:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-14T09:45:48,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T09:45:48,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-14T09:45:48,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.0030 sec 2024-11-14T09:45:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:45:48,911 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:48,911 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-14T09:45:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:45:48,913 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:48,913 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:48,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:45:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:49,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:45:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:49,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d5b8f55577f24998835f4ef9ed0de41f is 50, key is test_row_0/A:col10/1731577548345/Put/seqid=0 2024-11-14T09:45:49,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742267_1443 (size=14741) 2024-11-14T09:45:49,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577609104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:49,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:49,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:49,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577609220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:45:49,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d5b8f55577f24998835f4ef9ed0de41f 2024-11-14T09:45:49,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/83437e11a1ec4e12b057904c640237bc is 50, key is test_row_0/B:col10/1731577548345/Put/seqid=0 2024-11-14T09:45:49,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742268_1444 (size=12301) 2024-11-14T09:45:49,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577609441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:49,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:49,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:49,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:45:49,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577609760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:49,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/83437e11a1ec4e12b057904c640237bc 2024-11-14T09:45:49,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/9818fb6e45c64cf88e5753701f6e1a51 is 50, key is test_row_0/C:col10/1731577548345/Put/seqid=0 2024-11-14T09:45:49,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742269_1445 (size=12301) 2024-11-14T09:45:49,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:49,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:49,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:45:50,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/9818fb6e45c64cf88e5753701f6e1a51 2024-11-14T09:45:50,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/d5b8f55577f24998835f4ef9ed0de41f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d5b8f55577f24998835f4ef9ed0de41f 2024-11-14T09:45:50,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d5b8f55577f24998835f4ef9ed0de41f, entries=200, sequenceid=394, filesize=14.4 K 2024-11-14T09:45:50,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/83437e11a1ec4e12b057904c640237bc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/83437e11a1ec4e12b057904c640237bc 2024-11-14T09:45:50,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/83437e11a1ec4e12b057904c640237bc, entries=150, sequenceid=394, filesize=12.0 K 2024-11-14T09:45:50,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/9818fb6e45c64cf88e5753701f6e1a51 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/9818fb6e45c64cf88e5753701f6e1a51 2024-11-14T09:45:50,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/9818fb6e45c64cf88e5753701f6e1a51, entries=150, sequenceid=394, filesize=12.0 K 2024-11-14T09:45:50,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1248ms, sequenceid=394, compaction requested=true 2024-11-14T09:45:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:50,276 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:50,276 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:50,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54562 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/B is initiating minor compaction (all files) 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/A is initiating minor compaction (all files) 2024-11-14T09:45:50,277 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/A in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,277 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/B in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,277 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/74867f0dbe8d4ec4a9db8f170927444f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e6f463260a0640e3b8f1ae3e6ed19be7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c19b486d86a545158849ee4f024fd779, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d5b8f55577f24998835f4ef9ed0de41f] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=53.3 K 2024-11-14T09:45:50,277 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/be103d73c0234bb18a5238aaa6ed0648, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a72b1a3ab06b47898a2bc3a651a8d806, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/49daa4fe9d354aa8b36e6f8437b21ac1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/83437e11a1ec4e12b057904c640237bc] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=48.5 K 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74867f0dbe8d4ec4a9db8f170927444f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577544540 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting be103d73c0234bb18a5238aaa6ed0648, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577544540 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f463260a0640e3b8f1ae3e6ed19be7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731577545818 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a72b1a3ab06b47898a2bc3a651a8d806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731577545818 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19b486d86a545158849ee4f024fd779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731577547082 2024-11-14T09:45:50,277 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 49daa4fe9d354aa8b36e6f8437b21ac1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731577547082 2024-11-14T09:45:50,278 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b8f55577f24998835f4ef9ed0de41f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731577548345 2024-11-14T09:45:50,278 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 83437e11a1ec4e12b057904c640237bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731577548345 2024-11-14T09:45:50,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:45:50,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:50,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:50,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:50,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:50,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:50,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:50,286 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#B#compaction#378 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:50,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:50,286 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#A#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:50,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,286 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/2ce38731b14c4a02830401d217503693 is 50, key is test_row_0/B:col10/1731577548345/Put/seqid=0 2024-11-14T09:45:50,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:50,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,286 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c09cd8d4a04b4a01bbdbe1d6e6be0888 is 50, key is test_row_0/A:col10/1731577548345/Put/seqid=0 2024-11-14T09:45:50,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/dfe28e46e02c4e149d03bff9bc042030 is 50, key is test_row_0/A:col10/1731577549103/Put/seqid=0 2024-11-14T09:45:50,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742270_1446 (size=12915) 2024-11-14T09:45:50,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742271_1447 (size=12915) 2024-11-14T09:45:50,313 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/2ce38731b14c4a02830401d217503693 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2ce38731b14c4a02830401d217503693 2024-11-14T09:45:50,316 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/c09cd8d4a04b4a01bbdbe1d6e6be0888 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c09cd8d4a04b4a01bbdbe1d6e6be0888 2024-11-14T09:45:50,322 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/B of 9542cc54aea5ce4e3db1d4ac48e4216a into 2ce38731b14c4a02830401d217503693(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:50,322 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:50,322 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/B, priority=12, startTime=1731577550276; duration=0sec 2024-11-14T09:45:50,322 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:50,322 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:B 2024-11-14T09:45:50,322 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:50,326 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:50,327 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/C is initiating minor compaction (all files) 2024-11-14T09:45:50,327 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/C in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,327 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/85914010ec9b458692542b820328a652, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/b405a4a818c34e20ace1d77a03aad412, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f772217da4734d62bcf3b935b0351531, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/9818fb6e45c64cf88e5753701f6e1a51] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=48.5 K 2024-11-14T09:45:50,327 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 85914010ec9b458692542b820328a652, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577544540 2024-11-14T09:45:50,328 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b405a4a818c34e20ace1d77a03aad412, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731577545818 2024-11-14T09:45:50,328 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f772217da4734d62bcf3b935b0351531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731577547082 2024-11-14T09:45:50,328 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9818fb6e45c64cf88e5753701f6e1a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731577548345 2024-11-14T09:45:50,329 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/A of 9542cc54aea5ce4e3db1d4ac48e4216a into c09cd8d4a04b4a01bbdbe1d6e6be0888(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:50,329 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:50,329 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/A, priority=12, startTime=1731577550276; duration=0sec 2024-11-14T09:45:50,329 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:50,329 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:A 2024-11-14T09:45:50,337 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:50,338 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/bf1193a0fb9441b597b4fd1925bbb042 is 50, key is test_row_0/C:col10/1731577548345/Put/seqid=0 2024-11-14T09:45:50,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742272_1448 (size=14741) 2024-11-14T09:45:50,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742273_1449 (size=12915) 2024-11-14T09:45:50,358 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/bf1193a0fb9441b597b4fd1925bbb042 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/bf1193a0fb9441b597b4fd1925bbb042 2024-11-14T09:45:50,361 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/C of 9542cc54aea5ce4e3db1d4ac48e4216a into bf1193a0fb9441b597b4fd1925bbb042(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:50,361 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:50,361 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/C, priority=12, startTime=1731577550276; duration=0sec 2024-11-14T09:45:50,362 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:50,362 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:C 2024-11-14T09:45:50,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:50,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 348 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577610391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:50,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:50,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:50,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 349 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577610500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:50,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:50,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:50,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 350 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577610710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:50,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/dfe28e46e02c4e149d03bff9bc042030 2024-11-14T09:45:50,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:50,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:50,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/81d321fa1f774346b5f898b16d8e5624 is 50, key is test_row_0/B:col10/1731577549103/Put/seqid=0 2024-11-14T09:45:50,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742274_1450 (size=12301) 2024-11-14T09:45:50,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/81d321fa1f774346b5f898b16d8e5624 2024-11-14T09:45:50,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/4123867c2a1c4b76b875660533d9d6c2 is 50, key is test_row_0/C:col10/1731577549103/Put/seqid=0 2024-11-14T09:45:50,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742275_1451 (size=12301) 2024-11-14T09:45:50,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:50,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:50,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:50,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:51,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577611020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:51,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:51,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:51,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:51,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:45:51,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/4123867c2a1c4b76b875660533d9d6c2 2024-11-14T09:45:51,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/dfe28e46e02c4e149d03bff9bc042030 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/dfe28e46e02c4e149d03bff9bc042030 2024-11-14T09:45:51,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/dfe28e46e02c4e149d03bff9bc042030, entries=200, sequenceid=410, filesize=14.4 K 2024-11-14T09:45:51,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/81d321fa1f774346b5f898b16d8e5624 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/81d321fa1f774346b5f898b16d8e5624 2024-11-14T09:45:51,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/81d321fa1f774346b5f898b16d8e5624, entries=150, sequenceid=410, filesize=12.0 K 2024-11-14T09:45:51,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/4123867c2a1c4b76b875660533d9d6c2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4123867c2a1c4b76b875660533d9d6c2 2024-11-14T09:45:51,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4123867c2a1c4b76b875660533d9d6c2, entries=150, sequenceid=410, filesize=12.0 K 2024-11-14T09:45:51,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9542cc54aea5ce4e3db1d4ac48e4216a in 892ms, sequenceid=410, compaction requested=false 2024-11-14T09:45:51,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:51,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:45:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:51,202 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T09:45:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:51,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:51,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e599b42c17a0456ca09ea7dfbabccc1b is 50, key is test_row_0/A:col10/1731577550381/Put/seqid=0 2024-11-14T09:45:51,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742276_1452 (size=12301) 2024-11-14T09:45:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:51,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 365 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577611601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:51,618 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e599b42c17a0456ca09ea7dfbabccc1b 2024-11-14T09:45:51,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/484d00a96ec74bc38954bcff0c7bb70a is 50, key is test_row_0/B:col10/1731577550381/Put/seqid=0 2024-11-14T09:45:51,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742277_1453 (size=12301) 2024-11-14T09:45:51,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577611710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:51,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 367 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577611931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:52,028 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/484d00a96ec74bc38954bcff0c7bb70a 2024-11-14T09:45:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/dddc0afd31e843768e811216ce177310 is 50, key is test_row_0/C:col10/1731577550381/Put/seqid=0 2024-11-14T09:45:52,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742278_1454 (size=12301) 2024-11-14T09:45:52,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 368 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577612240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:52,440 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/dddc0afd31e843768e811216ce177310 2024-11-14T09:45:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/e599b42c17a0456ca09ea7dfbabccc1b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e599b42c17a0456ca09ea7dfbabccc1b 2024-11-14T09:45:52,446 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e599b42c17a0456ca09ea7dfbabccc1b, entries=150, sequenceid=429, filesize=12.0 K 2024-11-14T09:45:52,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/484d00a96ec74bc38954bcff0c7bb70a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/484d00a96ec74bc38954bcff0c7bb70a 2024-11-14T09:45:52,450 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/484d00a96ec74bc38954bcff0c7bb70a, entries=150, sequenceid=429, filesize=12.0 K 2024-11-14T09:45:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/dddc0afd31e843768e811216ce177310 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/dddc0afd31e843768e811216ce177310 2024-11-14T09:45:52,453 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/dddc0afd31e843768e811216ce177310, entries=150, sequenceid=429, filesize=12.0 K 2024-11-14T09:45:52,454 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1252ms, sequenceid=429, compaction requested=true 2024-11-14T09:45:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:52,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-14T09:45:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-14T09:45:52,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T09:45:52,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5420 sec 2024-11-14T09:45:52,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.5460 sec 2024-11-14T09:45:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:52,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T09:45:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:52,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:52,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/7148e6840d8b49028ea1886822f8897e is 50, key is test_row_0/A:col10/1731577551600/Put/seqid=0 2024-11-14T09:45:52,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742279_1455 (size=14741) 2024-11-14T09:45:52,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577612854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 383 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577612970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:45:53,081 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:53,081 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:45:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T09:45:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:45:53,082 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:45:53,083 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:45:53,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:45:53,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/7148e6840d8b49028ea1886822f8897e 2024-11-14T09:45:53,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/119dd820d2d4414495733cc7ba3d625a is 50, key is test_row_0/B:col10/1731577551600/Put/seqid=0 2024-11-14T09:45:53,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742280_1456 (size=12301) 2024-11-14T09:45:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 384 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577613180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:45:53,215 INFO [Thread-1794 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1794. 2024-11-14T09:45:53,215 DEBUG [Thread-1794 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:53,215 INFO [Thread-1800 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1800. 2024-11-14T09:45:53,215 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,215 DEBUG [Thread-1800 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:53,215 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,215 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,215 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:53,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:53,219 INFO [Thread-1796 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1796. 2024-11-14T09:45:53,219 DEBUG [Thread-1796 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:53,219 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,219 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:53,226 INFO [Thread-1798 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1798. 2024-11-14T09:45:53,226 DEBUG [Thread-1798 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:53,226 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,226 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:53,227 INFO [Thread-1792 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1792. 2024-11-14T09:45:53,227 DEBUG [Thread-1792 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:53,227 DEBUG [Thread-1792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,227 DEBUG [Thread-1792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:53,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:53,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:45:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:45:53,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:45:53,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:53,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42782 deadline: 1731577613501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:53,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:53,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:45:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42750 deadline: 1731577613521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:45:53,522 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in 9542cc54aea5ce4e3db1d4ac48e4216a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18338 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:45:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:45:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., hostname=defc576eb6b7,42511,1731577408298, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:45:53,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:45:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/119dd820d2d4414495733cc7ba3d625a 2024-11-14T09:45:53,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a9d7fc2b56ed40a49b113600e54cb15c is 50, key is test_row_0/C:col10/1731577551600/Put/seqid=0 2024-11-14T09:45:53,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742281_1457 (size=12301) 2024-11-14T09:45:53,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:45:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:45:53,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:45:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:53,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:45:53,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a9d7fc2b56ed40a49b113600e54cb15c 2024-11-14T09:45:53,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/7148e6840d8b49028ea1886822f8897e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7148e6840d8b49028ea1886822f8897e 2024-11-14T09:45:53,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7148e6840d8b49028ea1886822f8897e, entries=200, sequenceid=446, filesize=14.4 K 2024-11-14T09:45:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/119dd820d2d4414495733cc7ba3d625a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/119dd820d2d4414495733cc7ba3d625a 2024-11-14T09:45:53,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/119dd820d2d4414495733cc7ba3d625a, entries=150, sequenceid=446, filesize=12.0 K 2024-11-14T09:45:53,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a9d7fc2b56ed40a49b113600e54cb15c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a9d7fc2b56ed40a49b113600e54cb15c 2024-11-14T09:45:54,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a9d7fc2b56ed40a49b113600e54cb15c, entries=150, sequenceid=446, filesize=12.0 K 2024-11-14T09:45:54,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1238ms, sequenceid=446, compaction requested=true 2024-11-14T09:45:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:45:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:45:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:54,001 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9542cc54aea5ce4e3db1d4ac48e4216a:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:45:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:54,001 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:54,002 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:54,002 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:54,002 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/A is initiating minor compaction (all files) 2024-11-14T09:45:54,002 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/B is initiating minor compaction (all files) 2024-11-14T09:45:54,002 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/A in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:54,002 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/B in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:54,002 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c09cd8d4a04b4a01bbdbe1d6e6be0888, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/dfe28e46e02c4e149d03bff9bc042030, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e599b42c17a0456ca09ea7dfbabccc1b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7148e6840d8b49028ea1886822f8897e] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=53.4 K 2024-11-14T09:45:54,002 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2ce38731b14c4a02830401d217503693, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/81d321fa1f774346b5f898b16d8e5624, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/484d00a96ec74bc38954bcff0c7bb70a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/119dd820d2d4414495733cc7ba3d625a] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=48.7 K 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c09cd8d4a04b4a01bbdbe1d6e6be0888, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731577548345 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce38731b14c4a02830401d217503693, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731577548345 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe28e46e02c4e149d03bff9bc042030, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731577549055 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d321fa1f774346b5f898b16d8e5624, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731577549055 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting e599b42c17a0456ca09ea7dfbabccc1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731577550372 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 484d00a96ec74bc38954bcff0c7bb70a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731577550372 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7148e6840d8b49028ea1886822f8897e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731577551570 2024-11-14T09:45:54,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 119dd820d2d4414495733cc7ba3d625a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731577551589 2024-11-14T09:45:54,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:45:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:54,008 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T09:45:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:45:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:45:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:45:54,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:45:54,011 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:54,011 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:45:54,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. as already flushing 2024-11-14T09:45:54,011 INFO [Thread-1785 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1785. 2024-11-14T09:45:54,011 DEBUG [Thread-1785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:54,011 DEBUG [Thread-1785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:54,011 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/1c9afc19f2f24affb3be4f16914b1f36 is 50, key is test_row_0/A:col10/1731577551600/Put/seqid=0 2024-11-14T09:45:54,011 DEBUG [Thread-1785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:54,011 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/58383ab7af644845a18ba7b14ef086d4 is 50, key is test_row_0/B:col10/1731577551600/Put/seqid=0 2024-11-14T09:45:54,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/7b29091f89db45ef90b6f108397c46e2 is 50, key is test_row_0/A:col10/1731577552835/Put/seqid=0 2024-11-14T09:45:54,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742283_1459 (size=13051) 2024-11-14T09:45:54,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742282_1458 (size=13051) 2024-11-14T09:45:54,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742284_1460 (size=12301) 2024-11-14T09:45:54,019 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/7b29091f89db45ef90b6f108397c46e2 2024-11-14T09:45:54,020 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/58383ab7af644845a18ba7b14ef086d4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/58383ab7af644845a18ba7b14ef086d4 2024-11-14T09:45:54,020 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/1c9afc19f2f24affb3be4f16914b1f36 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/1c9afc19f2f24affb3be4f16914b1f36 2024-11-14T09:45:54,024 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/A of 9542cc54aea5ce4e3db1d4ac48e4216a into 1c9afc19f2f24affb3be4f16914b1f36(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:54,024 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/B of 9542cc54aea5ce4e3db1d4ac48e4216a into 58383ab7af644845a18ba7b14ef086d4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:54,024 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/A, priority=12, startTime=1731577554001; duration=0sec 2024-11-14T09:45:54,024 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/B, priority=12, startTime=1731577554001; duration=0sec 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:A 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:B 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:45:54,024 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:45:54,025 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): 9542cc54aea5ce4e3db1d4ac48e4216a/C is initiating minor compaction (all files) 2024-11-14T09:45:54,025 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9542cc54aea5ce4e3db1d4ac48e4216a/C in TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:54,025 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/bf1193a0fb9441b597b4fd1925bbb042, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4123867c2a1c4b76b875660533d9d6c2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/dddc0afd31e843768e811216ce177310, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a9d7fc2b56ed40a49b113600e54cb15c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp, totalSize=48.7 K 2024-11-14T09:45:54,025 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1193a0fb9441b597b4fd1925bbb042, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731577548345 2024-11-14T09:45:54,025 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4123867c2a1c4b76b875660533d9d6c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731577549055 2024-11-14T09:45:54,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/d04dd61b77654a8083865b44430cc88e is 50, key is test_row_0/B:col10/1731577552835/Put/seqid=0 2024-11-14T09:45:54,025 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dddc0afd31e843768e811216ce177310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731577550372 2024-11-14T09:45:54,025 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d7fc2b56ed40a49b113600e54cb15c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731577551589 2024-11-14T09:45:54,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742285_1461 (size=12301) 2024-11-14T09:45:54,031 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9542cc54aea5ce4e3db1d4ac48e4216a#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:45:54,032 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a3ef711319334354b73cdb87f8579ab8 is 50, key is test_row_0/C:col10/1731577551600/Put/seqid=0 2024-11-14T09:45:54,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742286_1462 (size=13051) 2024-11-14T09:45:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:45:54,429 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/d04dd61b77654a8083865b44430cc88e 2024-11-14T09:45:54,442 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/a3ef711319334354b73cdb87f8579ab8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a3ef711319334354b73cdb87f8579ab8 2024-11-14T09:45:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/29f84ba555dd441ea41084472141d0e5 is 50, key is test_row_0/C:col10/1731577552835/Put/seqid=0 2024-11-14T09:45:54,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742287_1463 (size=12301) 2024-11-14T09:45:54,447 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9542cc54aea5ce4e3db1d4ac48e4216a/C of 9542cc54aea5ce4e3db1d4ac48e4216a into a3ef711319334354b73cdb87f8579ab8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:45:54,447 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:54,447 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., storeName=9542cc54aea5ce4e3db1d4ac48e4216a/C, priority=12, startTime=1731577554001; duration=0sec 2024-11-14T09:45:54,447 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:45:54,447 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9542cc54aea5ce4e3db1d4ac48e4216a:C 2024-11-14T09:45:54,847 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/29f84ba555dd441ea41084472141d0e5 2024-11-14T09:45:54,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/7b29091f89db45ef90b6f108397c46e2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7b29091f89db45ef90b6f108397c46e2 2024-11-14T09:45:54,859 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7b29091f89db45ef90b6f108397c46e2, entries=150, sequenceid=461, filesize=12.0 K 2024-11-14T09:45:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/d04dd61b77654a8083865b44430cc88e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/d04dd61b77654a8083865b44430cc88e 2024-11-14T09:45:54,864 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/d04dd61b77654a8083865b44430cc88e, entries=150, sequenceid=461, filesize=12.0 K 2024-11-14T09:45:54,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/29f84ba555dd441ea41084472141d0e5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29f84ba555dd441ea41084472141d0e5 2024-11-14T09:45:54,868 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29f84ba555dd441ea41084472141d0e5, entries=150, sequenceid=461, filesize=12.0 K 2024-11-14T09:45:54,869 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for 9542cc54aea5ce4e3db1d4ac48e4216a in 861ms, sequenceid=461, compaction requested=false 2024-11-14T09:45:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 9542cc54aea5ce4e3db1d4ac48e4216a: 2024-11-14T09:45:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:45:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T09:45:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T09:45:54,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T09:45:54,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7870 sec 2024-11-14T09:45:54,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.7920 sec 2024-11-14T09:45:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:45:55,231 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:45:56,643 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:45:57,654 INFO [Thread-1787 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1787. 2024-11-14T09:45:57,654 DEBUG [Thread-1787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:57,654 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:57,654 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:57,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:57,701 INFO [Thread-1783 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1783. 2024-11-14T09:45:57,701 INFO [Thread-1781 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1781. 2024-11-14T09:45:57,701 DEBUG [Thread-1783 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:57,701 DEBUG [Thread-1781 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:45:57,701 DEBUG [Thread-1783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:57,701 DEBUG [Thread-1781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:57,702 DEBUG [Thread-1783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:57,702 DEBUG [Thread-1781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:45:57,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:45:57,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:03,622 INFO [Thread-1789 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1789. 2024-11-14T09:46:03,622 DEBUG [Thread-1789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:03,623 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:03,623 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:03,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 330 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1234 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3702 rows 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1230 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3684 rows 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1235 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3699 rows 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1231 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3693 rows 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1233 2024-11-14T09:46:03,623 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3694 rows 2024-11-14T09:46:03,623 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:46:03,623 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:46:03,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:03,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:03,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:03,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:46:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:46:03,628 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577563628"}]},"ts":"1731577563628"} 2024-11-14T09:46:03,630 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:46:03,630 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:46:03,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:46:03,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, UNASSIGN}] 2024-11-14T09:46:03,634 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, UNASSIGN 2024-11-14T09:46:03,635 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=9542cc54aea5ce4e3db1d4ac48e4216a, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:03,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, UNASSIGN because future has completed 2024-11-14T09:46:03,637 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:46:03,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:46:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:46:03,789 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:46:03,789 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:46:03,790 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 9542cc54aea5ce4e3db1d4ac48e4216a, disabling compactions & flushes 2024-11-14T09:46:03,790 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:46:03,790 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:46:03,790 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. after waiting 0 ms 2024-11-14T09:46:03,790 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:46:03,790 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 9542cc54aea5ce4e3db1d4ac48e4216a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T09:46:03,791 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=A 2024-11-14T09:46:03,791 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:03,791 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=B 2024-11-14T09:46:03,791 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:03,791 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9542cc54aea5ce4e3db1d4ac48e4216a, store=C 2024-11-14T09:46:03,791 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:03,794 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/260bf1562aab403b950159274bfb01e6 is 50, key is test_row_0/A:col10/1731577563621/Put/seqid=0 2024-11-14T09:46:03,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742288_1464 (size=12301) 2024-11-14T09:46:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:46:04,197 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/260bf1562aab403b950159274bfb01e6 2024-11-14T09:46:04,203 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/eff764c635eb4c488ed4c2e386553484 is 50, key is test_row_0/B:col10/1731577563621/Put/seqid=0 2024-11-14T09:46:04,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742289_1465 (size=12301) 2024-11-14T09:46:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:46:04,607 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/eff764c635eb4c488ed4c2e386553484 2024-11-14T09:46:04,613 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/7b99b1e132ce4e06a74e10aca2cf12ce is 50, key is test_row_0/C:col10/1731577563621/Put/seqid=0 2024-11-14T09:46:04,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742290_1466 (size=12301) 2024-11-14T09:46:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:46:05,017 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/7b99b1e132ce4e06a74e10aca2cf12ce 2024-11-14T09:46:05,020 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/A/260bf1562aab403b950159274bfb01e6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/260bf1562aab403b950159274bfb01e6 2024-11-14T09:46:05,024 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/260bf1562aab403b950159274bfb01e6, entries=150, sequenceid=472, filesize=12.0 K 2024-11-14T09:46:05,024 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/B/eff764c635eb4c488ed4c2e386553484 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/eff764c635eb4c488ed4c2e386553484 2024-11-14T09:46:05,027 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/eff764c635eb4c488ed4c2e386553484, entries=150, sequenceid=472, filesize=12.0 K 2024-11-14T09:46:05,028 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/.tmp/C/7b99b1e132ce4e06a74e10aca2cf12ce as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7b99b1e132ce4e06a74e10aca2cf12ce 2024-11-14T09:46:05,031 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7b99b1e132ce4e06a74e10aca2cf12ce, entries=150, sequenceid=472, filesize=12.0 K 2024-11-14T09:46:05,032 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1242ms, sequenceid=472, compaction requested=true 2024-11-14T09:46:05,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/5be39f7461d047e48f71b328daef1439, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/8fd372071b4d4260a8af321360444d85, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/9fac04691cdd44fa93dab0fe64b65597, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/3cfdaaf3bcea4e35bb8787658d41f5bc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/4e1bdd6cd6574872bae942a6c0a2e489, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/17a8dc9078ff4fc68cf38fa937b0d1eb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/10c67b79d7ac4c649d942b817d436757, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/62fac6776bce49fc9c2756344c13a35c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d56973bcade94010aec7c182c92b7e27, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/f0a36db5728641d68f323a220e1902bd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e0ed8c565ad342a39a4630f340abe056, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c315313ad4b1456798e01d13957d40dc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d111730aa64642b98942912fca685ed7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/b0f72ad7e84140e7b9919ba52e486230, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/74867f0dbe8d4ec4a9db8f170927444f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e6f463260a0640e3b8f1ae3e6ed19be7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c19b486d86a545158849ee4f024fd779, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d5b8f55577f24998835f4ef9ed0de41f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c09cd8d4a04b4a01bbdbe1d6e6be0888, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/dfe28e46e02c4e149d03bff9bc042030, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e599b42c17a0456ca09ea7dfbabccc1b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7148e6840d8b49028ea1886822f8897e] to archive 2024-11-14T09:46:05,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:46:05,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/5be39f7461d047e48f71b328daef1439 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/5be39f7461d047e48f71b328daef1439 2024-11-14T09:46:05,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/8fd372071b4d4260a8af321360444d85 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/8fd372071b4d4260a8af321360444d85 2024-11-14T09:46:05,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/9fac04691cdd44fa93dab0fe64b65597 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/9fac04691cdd44fa93dab0fe64b65597 2024-11-14T09:46:05,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/3cfdaaf3bcea4e35bb8787658d41f5bc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/3cfdaaf3bcea4e35bb8787658d41f5bc 2024-11-14T09:46:05,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/4e1bdd6cd6574872bae942a6c0a2e489 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/4e1bdd6cd6574872bae942a6c0a2e489 2024-11-14T09:46:05,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/17a8dc9078ff4fc68cf38fa937b0d1eb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/17a8dc9078ff4fc68cf38fa937b0d1eb 2024-11-14T09:46:05,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/10c67b79d7ac4c649d942b817d436757 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/10c67b79d7ac4c649d942b817d436757 2024-11-14T09:46:05,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/62fac6776bce49fc9c2756344c13a35c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/62fac6776bce49fc9c2756344c13a35c 2024-11-14T09:46:05,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d56973bcade94010aec7c182c92b7e27 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d56973bcade94010aec7c182c92b7e27 2024-11-14T09:46:05,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/f0a36db5728641d68f323a220e1902bd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/f0a36db5728641d68f323a220e1902bd 2024-11-14T09:46:05,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e0ed8c565ad342a39a4630f340abe056 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e0ed8c565ad342a39a4630f340abe056 2024-11-14T09:46:05,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c315313ad4b1456798e01d13957d40dc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c315313ad4b1456798e01d13957d40dc 2024-11-14T09:46:05,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d111730aa64642b98942912fca685ed7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d111730aa64642b98942912fca685ed7 2024-11-14T09:46:05,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/b0f72ad7e84140e7b9919ba52e486230 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/b0f72ad7e84140e7b9919ba52e486230 2024-11-14T09:46:05,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/74867f0dbe8d4ec4a9db8f170927444f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/74867f0dbe8d4ec4a9db8f170927444f 2024-11-14T09:46:05,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e6f463260a0640e3b8f1ae3e6ed19be7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e6f463260a0640e3b8f1ae3e6ed19be7 2024-11-14T09:46:05,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c19b486d86a545158849ee4f024fd779 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c19b486d86a545158849ee4f024fd779 2024-11-14T09:46:05,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d5b8f55577f24998835f4ef9ed0de41f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/d5b8f55577f24998835f4ef9ed0de41f 2024-11-14T09:46:05,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c09cd8d4a04b4a01bbdbe1d6e6be0888 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/c09cd8d4a04b4a01bbdbe1d6e6be0888 2024-11-14T09:46:05,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/dfe28e46e02c4e149d03bff9bc042030 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/dfe28e46e02c4e149d03bff9bc042030 2024-11-14T09:46:05,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e599b42c17a0456ca09ea7dfbabccc1b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/e599b42c17a0456ca09ea7dfbabccc1b 2024-11-14T09:46:05,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7148e6840d8b49028ea1886822f8897e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7148e6840d8b49028ea1886822f8897e 2024-11-14T09:46:05,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f657e256efc24004bbbfb63292d37e58, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2a72cf921cf74adeaa59ce005d71b30c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/0fe2c609e88b4f12b2a5a2799725dcda, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/7fd04601491f4a2d80968a01911e1e37, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/82e9269e9648466f9c0a61a41ad340b8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/b4d21444452048508441a81f98a2eae2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3ca2f6e30c0f4190830c893e863af9ff, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/e3ee48d39ca14491a6aec4cee1353088, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f4b9190ba1c04c50b3622a32576f0710, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3e391c3c8da34cc583ac61e8a6efdb45, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/96fc2decea8a453ea5fa38ef138617c7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/aa7d50a02bf94286a6d02e6c3b57324f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a089655d13524fbc888341f718473e36, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/be103d73c0234bb18a5238aaa6ed0648, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/48d84de56ee643e38c5fec0cfcaff9ec, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a72b1a3ab06b47898a2bc3a651a8d806, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/49daa4fe9d354aa8b36e6f8437b21ac1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2ce38731b14c4a02830401d217503693, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/83437e11a1ec4e12b057904c640237bc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/81d321fa1f774346b5f898b16d8e5624, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/484d00a96ec74bc38954bcff0c7bb70a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/119dd820d2d4414495733cc7ba3d625a] to archive 2024-11-14T09:46:05,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:46:05,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f657e256efc24004bbbfb63292d37e58 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f657e256efc24004bbbfb63292d37e58 2024-11-14T09:46:05,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2a72cf921cf74adeaa59ce005d71b30c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2a72cf921cf74adeaa59ce005d71b30c 2024-11-14T09:46:05,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/0fe2c609e88b4f12b2a5a2799725dcda to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/0fe2c609e88b4f12b2a5a2799725dcda 2024-11-14T09:46:05,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/7fd04601491f4a2d80968a01911e1e37 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/7fd04601491f4a2d80968a01911e1e37 2024-11-14T09:46:05,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/82e9269e9648466f9c0a61a41ad340b8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/82e9269e9648466f9c0a61a41ad340b8 2024-11-14T09:46:05,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/b4d21444452048508441a81f98a2eae2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/b4d21444452048508441a81f98a2eae2 2024-11-14T09:46:05,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3ca2f6e30c0f4190830c893e863af9ff to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3ca2f6e30c0f4190830c893e863af9ff 2024-11-14T09:46:05,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/e3ee48d39ca14491a6aec4cee1353088 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/e3ee48d39ca14491a6aec4cee1353088 2024-11-14T09:46:05,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f4b9190ba1c04c50b3622a32576f0710 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/f4b9190ba1c04c50b3622a32576f0710 2024-11-14T09:46:05,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3e391c3c8da34cc583ac61e8a6efdb45 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/3e391c3c8da34cc583ac61e8a6efdb45 2024-11-14T09:46:05,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/96fc2decea8a453ea5fa38ef138617c7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/96fc2decea8a453ea5fa38ef138617c7 2024-11-14T09:46:05,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/aa7d50a02bf94286a6d02e6c3b57324f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/aa7d50a02bf94286a6d02e6c3b57324f 2024-11-14T09:46:05,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a089655d13524fbc888341f718473e36 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a089655d13524fbc888341f718473e36 2024-11-14T09:46:05,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/be103d73c0234bb18a5238aaa6ed0648 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/be103d73c0234bb18a5238aaa6ed0648 2024-11-14T09:46:05,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/48d84de56ee643e38c5fec0cfcaff9ec to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/48d84de56ee643e38c5fec0cfcaff9ec 2024-11-14T09:46:05,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a72b1a3ab06b47898a2bc3a651a8d806 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/a72b1a3ab06b47898a2bc3a651a8d806 2024-11-14T09:46:05,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/49daa4fe9d354aa8b36e6f8437b21ac1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/49daa4fe9d354aa8b36e6f8437b21ac1 2024-11-14T09:46:05,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2ce38731b14c4a02830401d217503693 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/2ce38731b14c4a02830401d217503693 2024-11-14T09:46:05,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/83437e11a1ec4e12b057904c640237bc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/83437e11a1ec4e12b057904c640237bc 2024-11-14T09:46:05,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/81d321fa1f774346b5f898b16d8e5624 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/81d321fa1f774346b5f898b16d8e5624 2024-11-14T09:46:05,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/484d00a96ec74bc38954bcff0c7bb70a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/484d00a96ec74bc38954bcff0c7bb70a 2024-11-14T09:46:05,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/119dd820d2d4414495733cc7ba3d625a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/119dd820d2d4414495733cc7ba3d625a 2024-11-14T09:46:05,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e9b1d975911f48d4800902093bd7f2e9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4b15a917cb6443449128fa04de080e5d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a7bbeb28a8994d69bec904e001c7f454, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7555f9867d4e4216a79e849389636426, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1094f6ecf9b5486bb23878dfcf096171, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f9e5bf760de74820864993d3d58b7bb8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1b48c3e6f0e94e738965cf4d79da2a70, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e0f4cf4374d44e89b77041c8ed9953c5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/0e5456d26c98444baf810071be08230a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/615268a31fde479baab788d60ef781aa, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/47c15279a3b640958918893580ed7bdd, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/727fd180a52b4fc5b26826d8561fa1e8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a5c2306f17a44530b7382ed4729838b1, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/85914010ec9b458692542b820328a652, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29ad2a25ec024d619946c8804b721e1d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/b405a4a818c34e20ace1d77a03aad412, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f772217da4734d62bcf3b935b0351531, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/bf1193a0fb9441b597b4fd1925bbb042, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/9818fb6e45c64cf88e5753701f6e1a51, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4123867c2a1c4b76b875660533d9d6c2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/dddc0afd31e843768e811216ce177310, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a9d7fc2b56ed40a49b113600e54cb15c] to archive 2024-11-14T09:46:05,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:46:05,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e9b1d975911f48d4800902093bd7f2e9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e9b1d975911f48d4800902093bd7f2e9 2024-11-14T09:46:05,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4b15a917cb6443449128fa04de080e5d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4b15a917cb6443449128fa04de080e5d 2024-11-14T09:46:05,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a7bbeb28a8994d69bec904e001c7f454 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a7bbeb28a8994d69bec904e001c7f454 2024-11-14T09:46:05,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7555f9867d4e4216a79e849389636426 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7555f9867d4e4216a79e849389636426 2024-11-14T09:46:05,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1094f6ecf9b5486bb23878dfcf096171 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1094f6ecf9b5486bb23878dfcf096171 2024-11-14T09:46:05,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f9e5bf760de74820864993d3d58b7bb8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f9e5bf760de74820864993d3d58b7bb8 2024-11-14T09:46:05,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1b48c3e6f0e94e738965cf4d79da2a70 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/1b48c3e6f0e94e738965cf4d79da2a70 2024-11-14T09:46:05,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e0f4cf4374d44e89b77041c8ed9953c5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/e0f4cf4374d44e89b77041c8ed9953c5 2024-11-14T09:46:05,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/0e5456d26c98444baf810071be08230a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/0e5456d26c98444baf810071be08230a 2024-11-14T09:46:05,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/615268a31fde479baab788d60ef781aa to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/615268a31fde479baab788d60ef781aa 2024-11-14T09:46:05,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/47c15279a3b640958918893580ed7bdd to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/47c15279a3b640958918893580ed7bdd 2024-11-14T09:46:05,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/727fd180a52b4fc5b26826d8561fa1e8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/727fd180a52b4fc5b26826d8561fa1e8 2024-11-14T09:46:05,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a5c2306f17a44530b7382ed4729838b1 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a5c2306f17a44530b7382ed4729838b1 2024-11-14T09:46:05,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/85914010ec9b458692542b820328a652 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/85914010ec9b458692542b820328a652 2024-11-14T09:46:05,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29ad2a25ec024d619946c8804b721e1d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29ad2a25ec024d619946c8804b721e1d 2024-11-14T09:46:05,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/b405a4a818c34e20ace1d77a03aad412 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/b405a4a818c34e20ace1d77a03aad412 2024-11-14T09:46:05,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f772217da4734d62bcf3b935b0351531 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/f772217da4734d62bcf3b935b0351531 2024-11-14T09:46:05,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/bf1193a0fb9441b597b4fd1925bbb042 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/bf1193a0fb9441b597b4fd1925bbb042 2024-11-14T09:46:05,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/9818fb6e45c64cf88e5753701f6e1a51 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/9818fb6e45c64cf88e5753701f6e1a51 2024-11-14T09:46:05,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4123867c2a1c4b76b875660533d9d6c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/4123867c2a1c4b76b875660533d9d6c2 2024-11-14T09:46:05,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/dddc0afd31e843768e811216ce177310 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/dddc0afd31e843768e811216ce177310 2024-11-14T09:46:05,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a9d7fc2b56ed40a49b113600e54cb15c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a9d7fc2b56ed40a49b113600e54cb15c 2024-11-14T09:46:05,085 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/recovered.edits/475.seqid, newMaxSeqId=475, maxSeqId=1 2024-11-14T09:46:05,086 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. 2024-11-14T09:46:05,086 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 9542cc54aea5ce4e3db1d4ac48e4216a: Waiting for close lock at 1731577563790Running coprocessor pre-close hooks at 1731577563790Disabling compacts and flushes for region at 1731577563790Disabling writes for close at 1731577563790Obtaining lock to block concurrent updates at 1731577563790Preparing flush snapshotting stores in 9542cc54aea5ce4e3db1d4ac48e4216a at 1731577563790Finished memstore snapshotting TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731577563791 (+1 ms)Flushing stores of TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a. at 1731577563791Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/A: creating writer at 1731577563791Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/A: appending metadata at 1731577563793 (+2 ms)Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/A: closing flushed file at 1731577563793Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/B: creating writer at 1731577564201 (+408 ms)Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/B: appending metadata at 1731577564203 (+2 ms)Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/B: closing flushed file at 1731577564203Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/C: creating writer at 1731577564611 (+408 ms)Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/C: appending metadata at 1731577564613 (+2 ms)Flushing 9542cc54aea5ce4e3db1d4ac48e4216a/C: closing flushed file at 1731577564613Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66406134: reopening flushed file at 1731577565020 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@297426ee: reopening flushed file at 1731577565024 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71963dce: reopening flushed file at 1731577565027 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9542cc54aea5ce4e3db1d4ac48e4216a in 1242ms, sequenceid=472, compaction requested=true at 1731577565032 (+5 ms)Writing region close event to WAL at 1731577565082 (+50 ms)Running coprocessor post-close hooks at 1731577565085 (+3 ms)Closed at 1731577565086 (+1 ms) 2024-11-14T09:46:05,087 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:46:05,087 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=9542cc54aea5ce4e3db1d4ac48e4216a, regionState=CLOSED 2024-11-14T09:46:05,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:46:05,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T09:46:05,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 9542cc54aea5ce4e3db1d4ac48e4216a, server=defc576eb6b7,42511,1731577408298 in 1.4520 sec 2024-11-14T09:46:05,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-14T09:46:05,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9542cc54aea5ce4e3db1d4ac48e4216a, UNASSIGN in 1.4570 sec 2024-11-14T09:46:05,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T09:46:05,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4610 sec 2024-11-14T09:46:05,094 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577565093"}]},"ts":"1731577565093"} 2024-11-14T09:46:05,094 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:46:05,094 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:46:05,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4700 sec 2024-11-14T09:46:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:46:05,771 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:05,771 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:46:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:05,773 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:05,774 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:46:05,775 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:46:05,777 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/recovered.edits] 2024-11-14T09:46:05,780 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/1c9afc19f2f24affb3be4f16914b1f36 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/1c9afc19f2f24affb3be4f16914b1f36 2024-11-14T09:46:05,782 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/260bf1562aab403b950159274bfb01e6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/260bf1562aab403b950159274bfb01e6 2024-11-14T09:46:05,783 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7b29091f89db45ef90b6f108397c46e2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/A/7b29091f89db45ef90b6f108397c46e2 2024-11-14T09:46:05,786 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/58383ab7af644845a18ba7b14ef086d4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/58383ab7af644845a18ba7b14ef086d4 2024-11-14T09:46:05,787 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/d04dd61b77654a8083865b44430cc88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/d04dd61b77654a8083865b44430cc88e 2024-11-14T09:46:05,789 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/eff764c635eb4c488ed4c2e386553484 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/B/eff764c635eb4c488ed4c2e386553484 2024-11-14T09:46:05,791 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29f84ba555dd441ea41084472141d0e5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/29f84ba555dd441ea41084472141d0e5 2024-11-14T09:46:05,792 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7b99b1e132ce4e06a74e10aca2cf12ce to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/7b99b1e132ce4e06a74e10aca2cf12ce 2024-11-14T09:46:05,793 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a3ef711319334354b73cdb87f8579ab8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/C/a3ef711319334354b73cdb87f8579ab8 2024-11-14T09:46:05,796 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/recovered.edits/475.seqid to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a/recovered.edits/475.seqid 2024-11-14T09:46:05,796 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/9542cc54aea5ce4e3db1d4ac48e4216a 2024-11-14T09:46:05,796 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:46:05,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:05,800 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:46:05,803 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:46:05,804 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:05,804 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:46:05,804 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577565804"}]},"ts":"9223372036854775807"} 2024-11-14T09:46:05,807 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:46:05,807 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9542cc54aea5ce4e3db1d4ac48e4216a, NAME => 'TestAcidGuarantees,,1731577530999.9542cc54aea5ce4e3db1d4ac48e4216a.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:46:05,807 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:46:05,807 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577565807"}]},"ts":"9223372036854775807"} 2024-11-14T09:46:05,810 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:46:05,811 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:05,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-14T09:46:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:46:05,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:46:05,883 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:05,890 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 251), OpenFileDescriptor=443 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=328 (was 323) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3652 (was 3797) 2024-11-14T09:46:05,900 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=3651 2024-11-14T09:46:05,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:46:05,902 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:46:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:05,904 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:46:05,905 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:05,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-14T09:46:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T09:46:05,905 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:46:05,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742291_1467 (size=1061) 2024-11-14T09:46:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T09:46:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T09:46:06,314 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e 2024-11-14T09:46:06,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742292_1468 (size=53) 2024-11-14T09:46:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T09:46:06,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:46:06,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee11adb61130645457ff22474228d88e, disabling compactions & flushes 2024-11-14T09:46:06,720 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:06,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:06,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. after waiting 0 ms 2024-11-14T09:46:06,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:06,720 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:06,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee11adb61130645457ff22474228d88e: Waiting for close lock at 1731577566720Disabling compacts and flushes for region at 1731577566720Disabling writes for close at 1731577566720Writing region close event to WAL at 1731577566720Closed at 1731577566720 2024-11-14T09:46:06,721 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:46:06,721 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577566721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577566721"}]},"ts":"1731577566721"} 2024-11-14T09:46:06,723 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:46:06,724 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:46:06,724 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577566724"}]},"ts":"1731577566724"} 2024-11-14T09:46:06,725 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:46:06,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, ASSIGN}] 2024-11-14T09:46:06,727 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, ASSIGN 2024-11-14T09:46:06,728 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, ASSIGN; state=OFFLINE, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=false 2024-11-14T09:46:06,878 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:06,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, ASSIGN because future has completed 2024-11-14T09:46:06,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:46:07,035 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:07,035 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:46:07,036 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,036 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:46:07,036 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,036 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T09:46:07,041 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,042 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:46:07,043 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee11adb61130645457ff22474228d88e columnFamilyName A 2024-11-14T09:46:07,043 DEBUG [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:07,043 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(327): Store=ee11adb61130645457ff22474228d88e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:46:07,043 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,044 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:46:07,044 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee11adb61130645457ff22474228d88e columnFamilyName B 2024-11-14T09:46:07,044 DEBUG [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:07,045 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(327): Store=ee11adb61130645457ff22474228d88e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:46:07,045 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,046 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:46:07,047 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee11adb61130645457ff22474228d88e columnFamilyName C 2024-11-14T09:46:07,047 DEBUG [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:07,047 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(327): Store=ee11adb61130645457ff22474228d88e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:46:07,047 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:07,047 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,048 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,048 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,049 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,049 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,051 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:46:07,052 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,054 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:46:07,055 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened ee11adb61130645457ff22474228d88e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72793119, jitterRate=0.08470200002193451}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:46:07,055 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:07,055 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for ee11adb61130645457ff22474228d88e: Running coprocessor pre-open hook at 1731577567036Writing region info on filesystem at 1731577567036Initializing all the Stores at 1731577567041 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577567041Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577567041Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577567041Cleaning up temporary data from old regions at 1731577567050 (+9 ms)Running coprocessor post-open hooks at 1731577567055 (+5 ms)Region opened successfully at 1731577567055 2024-11-14T09:46:07,056 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., pid=126, masterSystemTime=1731577567033 2024-11-14T09:46:07,058 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:07,058 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:07,058 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:07,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:46:07,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T09:46:07,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 in 180 msec 2024-11-14T09:46:07,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-14T09:46:07,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, ASSIGN in 337 msec 2024-11-14T09:46:07,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:46:07,067 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577567067"}]},"ts":"1731577567067"} 2024-11-14T09:46:07,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:46:07,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:46:07,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-14T09:46:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T09:46:08,051 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:08,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:08,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:08,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:08,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658cf8c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:08,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:08,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a6117c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,056 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:08,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:08,059 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:46:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,33347,1731577407591 2024-11-14T09:46:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d85a684 2024-11-14T09:46:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:46:08,061 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:46:08,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:46:08,063 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:46:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:08,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742293_1469 (size=1097) 2024-11-14T09:46:08,476 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T09:46:08,476 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T09:46:08,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:46:08,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, REOPEN/MOVE}] 2024-11-14T09:46:08,482 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, REOPEN/MOVE 2024-11-14T09:46:08,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:08,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, REOPEN/MOVE because future has completed 2024-11-14T09:46:08,484 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:46:08,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:46:08,636 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,636 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:46:08,637 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing ee11adb61130645457ff22474228d88e, disabling compactions & flushes 2024-11-14T09:46:08,637 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,637 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,637 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. after waiting 0 ms 2024-11-14T09:46:08,637 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,640 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:46:08,641 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,641 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for ee11adb61130645457ff22474228d88e: Waiting for close lock at 1731577568637Running coprocessor pre-close hooks at 1731577568637Disabling compacts and flushes for region at 1731577568637Disabling writes for close at 1731577568637Writing region close event to WAL at 1731577568637Running coprocessor post-close hooks at 1731577568641 (+4 ms)Closed at 1731577568641 2024-11-14T09:46:08,641 WARN [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionServer(3299): Not adding moved region record: ee11adb61130645457ff22474228d88e to self. 2024-11-14T09:46:08,643 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,643 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=CLOSED 2024-11-14T09:46:08,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:46:08,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T09:46:08,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 in 162 msec 2024-11-14T09:46:08,648 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, REOPEN/MOVE; state=CLOSED, location=defc576eb6b7,42511,1731577408298; forceNewPlan=false, retain=true 2024-11-14T09:46:08,798 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=OPENING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:08,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, REOPEN/MOVE because future has completed 2024-11-14T09:46:08,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:46:08,955 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,955 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7752): Opening region: {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:46:08,955 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,956 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:46:08,956 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7794): checking encryption for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,956 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7797): checking classloading for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,957 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,958 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:46:08,958 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee11adb61130645457ff22474228d88e columnFamilyName A 2024-11-14T09:46:08,959 DEBUG [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:08,959 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(327): Store=ee11adb61130645457ff22474228d88e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:46:08,960 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,960 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:46:08,960 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee11adb61130645457ff22474228d88e columnFamilyName B 2024-11-14T09:46:08,960 DEBUG [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:08,961 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(327): Store=ee11adb61130645457ff22474228d88e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:46:08,961 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,961 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T09:46:08,962 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee11adb61130645457ff22474228d88e columnFamilyName C 2024-11-14T09:46:08,962 DEBUG [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:08,962 INFO [StoreOpener-ee11adb61130645457ff22474228d88e-1 {}] regionserver.HStore(327): Store=ee11adb61130645457ff22474228d88e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:46:08,962 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,962 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1038): replaying wal for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,963 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,964 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,965 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1048): stopping wal replay for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,965 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1060): Cleaning up temporary data for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,965 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:46:08,966 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1093): writing seq id for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,967 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1114): Opened ee11adb61130645457ff22474228d88e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63431072, jitterRate=-0.05480337142944336}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:46:08,967 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee11adb61130645457ff22474228d88e 2024-11-14T09:46:08,968 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1006): Region open journal for ee11adb61130645457ff22474228d88e: Running coprocessor pre-open hook at 1731577568956Writing region info on filesystem at 1731577568956Initializing all the Stores at 1731577568957 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577568957Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577568957Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577568957Cleaning up temporary data from old regions at 1731577568965 (+8 ms)Running coprocessor post-open hooks at 1731577568967 (+2 ms)Region opened successfully at 1731577568968 (+1 ms) 2024-11-14T09:46:08,969 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., pid=131, masterSystemTime=1731577568952 2024-11-14T09:46:08,971 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,971 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:08,971 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=OPEN, openSeqNum=5, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:08,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:46:08,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=129 2024-11-14T09:46:08,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 in 173 msec 2024-11-14T09:46:08,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-14T09:46:08,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, REOPEN/MOVE in 494 msec 2024-11-14T09:46:08,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T09:46:08,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-14T09:46:08,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-14T09:46:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:46:08,982 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:08,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:08,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:08,984 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d319267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:08,986 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@719ab69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:08,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:08,988 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7f6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:08,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:08,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426dec46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:08,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:08,992 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e5cde8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:08,993 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384edbde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:08,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:08,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7454c99f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:08,999 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3fa6db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,000 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:09,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:09,001 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa278b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:09,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9222cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:09,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:09,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c8bf46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:09,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f4a1cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:09,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:09,026 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:09,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20bcf33c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:09,028 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bd66e90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:09,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:09,033 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a151578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:09,035 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@487f45ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:09,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:09,042 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:09,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:09,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:09,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d312fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:09,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:09,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:09,043 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ac12f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,33347,-1 for getting cluster id 2024-11-14T09:46:09,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:46:09,049 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b85ece77-3b90-40fa-9bf4-8df5cb511dfe' 2024-11-14T09:46:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:46:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b85ece77-3b90-40fa-9bf4-8df5cb511dfe" 2024-11-14T09:46:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4bc4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,33347,-1] 2024-11-14T09:46:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:46:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:09,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:46:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e21329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:46:09,052 DEBUG [Thread-2140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,052 DEBUG [Thread-2138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,056 DEBUG [Thread-2127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,057 DEBUG [Thread-2131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,059 DEBUG [Thread-2127 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,059 DEBUG [Thread-2127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:09,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:46:09,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:09,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:09,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:09,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:09,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:09,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:09,080 DEBUG [Thread-2142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,088 DEBUG [Thread-2129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,097 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-11-14T09:46:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T09:46:09,100 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:09,101 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:09,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:09,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577569091, max timestamp=1731577569111Num uniques 859; 2024-11-14T09:46:09,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577569091, max timestamp=1731577569111Num uniques 859; 2024-11-14T09:46:09,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:46:09,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:46:09,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577569091, max timestamp=1731577569111Num uniques 859; 2024-11-14T09:46:09,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:46:09,116 DEBUG [Thread-2133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,132 DEBUG [Thread-2144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577629137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577629137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577629139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577629140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,148 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111441878a501a3f4f7e9e9afb9a9f7be51f_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577569063/Put/seqid=0 2024-11-14T09:46:09,184 DEBUG [Thread-2125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:46:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,42511,1731577408298, seqNum=-1] 2024-11-14T09:46:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:46:09,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:46:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5] 2024-11-14T09:46:09,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577629190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T09:46:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742294_1470 (size=12154) 2024-11-14T09:46:09,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T09:46:09,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:09,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577629251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577629251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577629251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577629251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577629301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T09:46:09,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:09,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T09:46:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577629460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577629461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577629461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577629461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577629510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T09:46:09,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:09,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,621 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:09,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111441878a501a3f4f7e9e9afb9a9f7be51f_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111441878a501a3f4f7e9e9afb9a9f7be51f_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:09,627 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b4862536059422e964bf60f7d748989, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:09,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b4862536059422e964bf60f7d748989 is 175, key is test_row_0/A:col10/1731577569063/Put/seqid=0 2024-11-14T09:46:09,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742295_1471 (size=30955) 2024-11-14T09:46:09,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T09:46:09,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:09,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T09:46:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577629770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577629770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577629770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577629780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577629820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:09,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T09:46:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:09,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:10,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T09:46:10,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:10,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:10,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:10,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:10,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b4862536059422e964bf60f7d748989 2024-11-14T09:46:10,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e6fbf14940cd44db8e2af9df9dfcc0b8 is 50, key is test_row_0/B:col10/1731577569063/Put/seqid=0 2024-11-14T09:46:10,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742296_1472 (size=12001) 2024-11-14T09:46:10,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e6fbf14940cd44db8e2af9df9dfcc0b8 2024-11-14T09:46:10,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8adb14780b784c8099149242e06a8180 is 50, key is test_row_0/C:col10/1731577569063/Put/seqid=0 2024-11-14T09:46:10,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742297_1473 (size=12001) 2024-11-14T09:46:10,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8adb14780b784c8099149242e06a8180 2024-11-14T09:46:10,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b4862536059422e964bf60f7d748989 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b4862536059422e964bf60f7d748989 2024-11-14T09:46:10,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b4862536059422e964bf60f7d748989, entries=150, sequenceid=14, filesize=30.2 K 2024-11-14T09:46:10,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e6fbf14940cd44db8e2af9df9dfcc0b8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e6fbf14940cd44db8e2af9df9dfcc0b8 2024-11-14T09:46:10,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e6fbf14940cd44db8e2af9df9dfcc0b8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:46:10,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8adb14780b784c8099149242e06a8180 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8adb14780b784c8099149242e06a8180 2024-11-14T09:46:10,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8adb14780b784c8099149242e06a8180, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:46:10,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ee11adb61130645457ff22474228d88e in 1054ms, sequenceid=14, compaction requested=false 2024-11-14T09:46:10,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:10,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-14T09:46:10,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:10,174 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-14T09:46:10,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:10,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ccdebcf5afdc42ee81e6f8885c65da22_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577569139/Put/seqid=0 2024-11-14T09:46:10,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742298_1474 (size=12154) 2024-11-14T09:46:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T09:46:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:10,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577630291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577630297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577630298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577630300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577630331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577630400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577630410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577630411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:10,589 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ccdebcf5afdc42ee81e6f8885c65da22_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ccdebcf5afdc42ee81e6f8885c65da22_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:10,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/ac30964097f44a38b55574b43dd9cd08, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:10,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/ac30964097f44a38b55574b43dd9cd08 is 175, key is test_row_0/A:col10/1731577569139/Put/seqid=0 2024-11-14T09:46:10,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742299_1475 (size=30955) 2024-11-14T09:46:10,605 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/ac30964097f44a38b55574b43dd9cd08 2024-11-14T09:46:10,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577630611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/26d140d6e2b54ea09c32e13bc785606b is 50, key is test_row_0/B:col10/1731577569139/Put/seqid=0 2024-11-14T09:46:10,621 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:46:10,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577630621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577630622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742300_1476 (size=12001) 2024-11-14T09:46:10,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577630920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577630931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:10,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577630940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,056 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/26d140d6e2b54ea09c32e13bc785606b 2024-11-14T09:46:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/49632e8cae5841fe923cf86e28929763 is 50, key is test_row_0/C:col10/1731577569139/Put/seqid=0 2024-11-14T09:46:11,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742301_1477 (size=12001) 2024-11-14T09:46:11,103 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/49632e8cae5841fe923cf86e28929763 2024-11-14T09:46:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/ac30964097f44a38b55574b43dd9cd08 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ac30964097f44a38b55574b43dd9cd08 2024-11-14T09:46:11,114 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ac30964097f44a38b55574b43dd9cd08, entries=150, sequenceid=48, filesize=30.2 K 2024-11-14T09:46:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/26d140d6e2b54ea09c32e13bc785606b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/26d140d6e2b54ea09c32e13bc785606b 2024-11-14T09:46:11,128 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/26d140d6e2b54ea09c32e13bc785606b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T09:46:11,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/49632e8cae5841fe923cf86e28929763 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/49632e8cae5841fe923cf86e28929763 2024-11-14T09:46:11,135 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/49632e8cae5841fe923cf86e28929763, entries=150, sequenceid=48, filesize=11.7 K 2024-11-14T09:46:11,137 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for ee11adb61130645457ff22474228d88e in 962ms, sequenceid=48, compaction requested=false 2024-11-14T09:46:11,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:11,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-11-14T09:46:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=133 2024-11-14T09:46:11,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-14T09:46:11,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-14T09:46:11,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 2.0440 sec 2024-11-14T09:46:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-14T09:46:11,250 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:11,251 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-11-14T09:46:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T09:46:11,252 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:11,253 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:11,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:11,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:46:11,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:11,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:11,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:11,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:11,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:11,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:11,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d67b50b9ffcd43b88b513e95adf4d14c_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577570281/Put/seqid=0 2024-11-14T09:46:11,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742302_1478 (size=14594) 2024-11-14T09:46:11,335 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:11,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d67b50b9ffcd43b88b513e95adf4d14c_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d67b50b9ffcd43b88b513e95adf4d14c_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:11,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/eceb93dc61a743088f758c26e45c1945, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:11,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/eceb93dc61a743088f758c26e45c1945 is 175, key is test_row_0/A:col10/1731577570281/Put/seqid=0 2024-11-14T09:46:11,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742303_1479 (size=39549) 2024-11-14T09:46:11,345 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/eceb93dc61a743088f758c26e45c1945 2024-11-14T09:46:11,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/2565e9bc02214640b4b867cb3112b30a is 50, key is test_row_0/B:col10/1731577570281/Put/seqid=0 2024-11-14T09:46:11,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742304_1480 (size=12001) 2024-11-14T09:46:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T09:46:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577631378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577631383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T09:46:11,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:11,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577631430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577631440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577631450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577631490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577631500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T09:46:11,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:11,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T09:46:11,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577631700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T09:46:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577631710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:11,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:11,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/2565e9bc02214640b4b867cb3112b30a 2024-11-14T09:46:11,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c9a8fc2fc83b4d6bb9c0e89329f01247 is 50, key is test_row_0/C:col10/1731577570281/Put/seqid=0 2024-11-14T09:46:11,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742305_1481 (size=12001) 2024-11-14T09:46:11,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T09:46:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:11,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T09:46:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577632010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T09:46:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:12,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:12,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577632020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c9a8fc2fc83b4d6bb9c0e89329f01247 2024-11-14T09:46:12,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T09:46:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:12,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/eceb93dc61a743088f758c26e45c1945 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/eceb93dc61a743088f758c26e45c1945 2024-11-14T09:46:12,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/eceb93dc61a743088f758c26e45c1945, entries=200, sequenceid=60, filesize=38.6 K 2024-11-14T09:46:12,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/2565e9bc02214640b4b867cb3112b30a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/2565e9bc02214640b4b867cb3112b30a 2024-11-14T09:46:12,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/2565e9bc02214640b4b867cb3112b30a, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:46:12,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c9a8fc2fc83b4d6bb9c0e89329f01247 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c9a8fc2fc83b4d6bb9c0e89329f01247 2024-11-14T09:46:12,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c9a8fc2fc83b4d6bb9c0e89329f01247, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T09:46:12,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ee11adb61130645457ff22474228d88e in 860ms, sequenceid=60, compaction requested=true 2024-11-14T09:46:12,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:12,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:12,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:12,184 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:12,184 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:12,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:12,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:12,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:12,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:12,186 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:12,186 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:12,186 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:12,186 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:12,186 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,186 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,186 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e6fbf14940cd44db8e2af9df9dfcc0b8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/26d140d6e2b54ea09c32e13bc785606b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/2565e9bc02214640b4b867cb3112b30a] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=35.2 K 2024-11-14T09:46:12,186 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b4862536059422e964bf60f7d748989, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ac30964097f44a38b55574b43dd9cd08, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/eceb93dc61a743088f758c26e45c1945] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=99.1 K 2024-11-14T09:46:12,186 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,186 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b4862536059422e964bf60f7d748989, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ac30964097f44a38b55574b43dd9cd08, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/eceb93dc61a743088f758c26e45c1945] 2024-11-14T09:46:12,187 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e6fbf14940cd44db8e2af9df9dfcc0b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577569063 2024-11-14T09:46:12,187 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b4862536059422e964bf60f7d748989, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577569063 2024-11-14T09:46:12,187 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d140d6e2b54ea09c32e13bc785606b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731577569129 2024-11-14T09:46:12,187 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac30964097f44a38b55574b43dd9cd08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731577569129 2024-11-14T09:46:12,187 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 2565e9bc02214640b4b867cb3112b30a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577570281 2024-11-14T09:46:12,188 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting eceb93dc61a743088f758c26e45c1945, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577570281 2024-11-14T09:46:12,193 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:12,194 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:12,194 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d58923686dad4b29bb83cf9335281f57_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:12,195 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/9b2058cddb60499d96b97709aacd6169 is 50, key is test_row_0/B:col10/1731577570281/Put/seqid=0 2024-11-14T09:46:12,197 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d58923686dad4b29bb83cf9335281f57_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:12,197 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d58923686dad4b29bb83cf9335281f57_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:12,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742306_1482 (size=12104) 2024-11-14T09:46:12,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742307_1483 (size=4469) 2024-11-14T09:46:12,206 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#409 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:12,207 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6723d7b0059e4dd69c96917aebeed90b is 175, key is test_row_0/A:col10/1731577570281/Put/seqid=0 2024-11-14T09:46:12,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742308_1484 (size=31058) 2024-11-14T09:46:12,218 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6723d7b0059e4dd69c96917aebeed90b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6723d7b0059e4dd69c96917aebeed90b 2024-11-14T09:46:12,222 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into 6723d7b0059e4dd69c96917aebeed90b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:12,222 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:12,222 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=13, startTime=1731577572184; duration=0sec 2024-11-14T09:46:12,222 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:12,222 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:12,222 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:12,223 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:12,223 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:12,223 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,223 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8adb14780b784c8099149242e06a8180, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/49632e8cae5841fe923cf86e28929763, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c9a8fc2fc83b4d6bb9c0e89329f01247] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=35.2 K 2024-11-14T09:46:12,224 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8adb14780b784c8099149242e06a8180, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577569063 2024-11-14T09:46:12,224 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49632e8cae5841fe923cf86e28929763, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731577569129 2024-11-14T09:46:12,224 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a8fc2fc83b4d6bb9c0e89329f01247, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577570281 2024-11-14T09:46:12,233 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:12,234 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/d474996986f34e1ebc97f80f78b801c5 is 50, key is test_row_0/C:col10/1731577570281/Put/seqid=0 2024-11-14T09:46:12,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742309_1485 (size=12104) 2024-11-14T09:46:12,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-14T09:46:12,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:12,322 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:46:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:12,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1212978e4df4d9fa1fca02ab8e08cc0_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577571382/Put/seqid=0 2024-11-14T09:46:12,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742310_1486 (size=12154) 2024-11-14T09:46:12,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:12,348 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b1212978e4df4d9fa1fca02ab8e08cc0_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1212978e4df4d9fa1fca02ab8e08cc0_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:12,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/06b112bcf7f947679c71928765a774f5, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:12,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/06b112bcf7f947679c71928765a774f5 is 175, key is test_row_0/A:col10/1731577571382/Put/seqid=0 2024-11-14T09:46:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742311_1487 (size=30955) 2024-11-14T09:46:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T09:46:12,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577632472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577632475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577632476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577632520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577632540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577632580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577632580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577632590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,607 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/9b2058cddb60499d96b97709aacd6169 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/9b2058cddb60499d96b97709aacd6169 2024-11-14T09:46:12,611 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into 9b2058cddb60499d96b97709aacd6169(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:12,611 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:12,611 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=13, startTime=1731577572184; duration=0sec 2024-11-14T09:46:12,611 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:12,611 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:12,642 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/d474996986f34e1ebc97f80f78b801c5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d474996986f34e1ebc97f80f78b801c5 2024-11-14T09:46:12,647 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into d474996986f34e1ebc97f80f78b801c5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:12,647 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:12,647 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=13, startTime=1731577572185; duration=0sec 2024-11-14T09:46:12,647 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:12,647 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:12,753 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/06b112bcf7f947679c71928765a774f5 2024-11-14T09:46:12,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/73aa7c509967459bb174a67b3b860fc6 is 50, key is test_row_0/B:col10/1731577571382/Put/seqid=0 2024-11-14T09:46:12,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742312_1488 (size=12001) 2024-11-14T09:46:12,771 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/73aa7c509967459bb174a67b3b860fc6 2024-11-14T09:46:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/7a84ab006670456e851a299a6328d248 is 50, key is test_row_0/C:col10/1731577571382/Put/seqid=0 2024-11-14T09:46:12,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742313_1489 (size=12001) 2024-11-14T09:46:12,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577632790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577632790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577632800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577633100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577633100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577633110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,186 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/7a84ab006670456e851a299a6328d248 2024-11-14T09:46:13,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/06b112bcf7f947679c71928765a774f5 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/06b112bcf7f947679c71928765a774f5 2024-11-14T09:46:13,200 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/06b112bcf7f947679c71928765a774f5, entries=150, sequenceid=81, filesize=30.2 K 2024-11-14T09:46:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/73aa7c509967459bb174a67b3b860fc6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/73aa7c509967459bb174a67b3b860fc6 2024-11-14T09:46:13,205 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/73aa7c509967459bb174a67b3b860fc6, entries=150, sequenceid=81, filesize=11.7 K 2024-11-14T09:46:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/7a84ab006670456e851a299a6328d248 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7a84ab006670456e851a299a6328d248 2024-11-14T09:46:13,209 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7a84ab006670456e851a299a6328d248, entries=150, sequenceid=81, filesize=11.7 K 2024-11-14T09:46:13,210 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ee11adb61130645457ff22474228d88e in 889ms, sequenceid=81, compaction requested=false 2024-11-14T09:46:13,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:13,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:13,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-11-14T09:46:13,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=135 2024-11-14T09:46:13,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-14T09:46:13,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-11-14T09:46:13,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 1.9630 sec 2024-11-14T09:46:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-14T09:46:13,400 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:13,400 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-11-14T09:46:13,402 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:46:13,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:13,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:46:13,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-14T09:46:13,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:13,556 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T09:46:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e482db9157cc48e9b271322c619ee2f1_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577572464/Put/seqid=0 2024-11-14T09:46:13,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742314_1490 (size=14594) 2024-11-14T09:46:13,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577633607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577633609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577633610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577633610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577633621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:46:13,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577633720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577633720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577633930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577633931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:13,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:13,987 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e482db9157cc48e9b271322c619ee2f1_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e482db9157cc48e9b271322c619ee2f1_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:13,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cd627391288b4450b7da8e7e8d639b57, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:13,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cd627391288b4450b7da8e7e8d639b57 is 175, key is test_row_0/A:col10/1731577572464/Put/seqid=0 2024-11-14T09:46:14,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742315_1491 (size=39549) 2024-11-14T09:46:14,023 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cd627391288b4450b7da8e7e8d639b57 2024-11-14T09:46:14,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f493eb50e33144159960288adac56969 is 50, key is test_row_0/B:col10/1731577572464/Put/seqid=0 2024-11-14T09:46:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:46:14,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742316_1492 (size=12001) 2024-11-14T09:46:14,044 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f493eb50e33144159960288adac56969 2024-11-14T09:46:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/9bc053e4e13447308a0150badd2cf6c2 is 50, key is test_row_0/C:col10/1731577572464/Put/seqid=0 2024-11-14T09:46:14,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742317_1493 (size=12001) 2024-11-14T09:46:14,055 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/9bc053e4e13447308a0150badd2cf6c2 2024-11-14T09:46:14,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cd627391288b4450b7da8e7e8d639b57 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cd627391288b4450b7da8e7e8d639b57 2024-11-14T09:46:14,066 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cd627391288b4450b7da8e7e8d639b57, entries=200, sequenceid=97, filesize=38.6 K 2024-11-14T09:46:14,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f493eb50e33144159960288adac56969 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f493eb50e33144159960288adac56969 2024-11-14T09:46:14,073 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f493eb50e33144159960288adac56969, entries=150, sequenceid=97, filesize=11.7 K 2024-11-14T09:46:14,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/9bc053e4e13447308a0150badd2cf6c2 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9bc053e4e13447308a0150badd2cf6c2 2024-11-14T09:46:14,077 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9bc053e4e13447308a0150badd2cf6c2, entries=150, sequenceid=97, filesize=11.7 K 2024-11-14T09:46:14,079 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ee11adb61130645457ff22474228d88e in 523ms, sequenceid=97, compaction requested=true 2024-11-14T09:46:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-11-14T09:46:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-11-14T09:46:14,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-14T09:46:14,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 677 msec 2024-11-14T09:46:14,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 687 msec 2024-11-14T09:46:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:14,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T09:46:14,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:14,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:14,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:14,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:14,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:14,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:14,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140faba77dfbcf44a68bf4ad2549ee9adb_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577574241/Put/seqid=0 2024-11-14T09:46:14,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742318_1494 (size=12154) 2024-11-14T09:46:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577634270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577634270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577634380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577634380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T09:46:14,540 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:14,541 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-14T09:46:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T09:46:14,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:14,544 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:14,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:14,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577634590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577634591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577634620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,626 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ee11adb61130645457ff22474228d88e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:46:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577634630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577634630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,634 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_1' in ee11adb61130645457ff22474228d88e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:46:14,634 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_2' in ee11adb61130645457ff22474228d88e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:46:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T09:46:14,668 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:14,674 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140faba77dfbcf44a68bf4ad2549ee9adb_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140faba77dfbcf44a68bf4ad2549ee9adb_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:14,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6e9f085eebcd42f1a917cb4138f1ea08, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:14,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6e9f085eebcd42f1a917cb4138f1ea08 is 175, key is test_row_0/A:col10/1731577574241/Put/seqid=0 2024-11-14T09:46:14,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-14T09:46:14,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:14,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:14,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:14,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742319_1495 (size=30955) 2024-11-14T09:46:14,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6e9f085eebcd42f1a917cb4138f1ea08 2024-11-14T09:46:14,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/692dc8cb64164b16b13c5e8a24cedd03 is 50, key is test_row_0/B:col10/1731577574241/Put/seqid=0 2024-11-14T09:46:14,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742320_1496 (size=12001) 2024-11-14T09:46:14,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/692dc8cb64164b16b13c5e8a24cedd03 2024-11-14T09:46:14,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/ef09a6fadfa241bf94fe46f89e65d25c is 50, key is test_row_0/C:col10/1731577574241/Put/seqid=0 2024-11-14T09:46:14,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742321_1497 (size=12001) 2024-11-14T09:46:14,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/ef09a6fadfa241bf94fe46f89e65d25c 2024-11-14T09:46:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6e9f085eebcd42f1a917cb4138f1ea08 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6e9f085eebcd42f1a917cb4138f1ea08 2024-11-14T09:46:14,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6e9f085eebcd42f1a917cb4138f1ea08, entries=150, sequenceid=116, filesize=30.2 K 2024-11-14T09:46:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/692dc8cb64164b16b13c5e8a24cedd03 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/692dc8cb64164b16b13c5e8a24cedd03 2024-11-14T09:46:14,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/692dc8cb64164b16b13c5e8a24cedd03, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:46:14,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/ef09a6fadfa241bf94fe46f89e65d25c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ef09a6fadfa241bf94fe46f89e65d25c 2024-11-14T09:46:14,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ef09a6fadfa241bf94fe46f89e65d25c, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:46:14,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for ee11adb61130645457ff22474228d88e in 588ms, sequenceid=116, compaction requested=true 2024-11-14T09:46:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:14,830 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:14,830 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:14,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:14,831 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:14,832 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:14,832 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,832 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/9b2058cddb60499d96b97709aacd6169, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/73aa7c509967459bb174a67b3b860fc6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f493eb50e33144159960288adac56969, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/692dc8cb64164b16b13c5e8a24cedd03] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=47.0 K 2024-11-14T09:46:14,834 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:14,834 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:14,834 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,834 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6723d7b0059e4dd69c96917aebeed90b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/06b112bcf7f947679c71928765a774f5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cd627391288b4450b7da8e7e8d639b57, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6e9f085eebcd42f1a917cb4138f1ea08] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=129.4 K 2024-11-14T09:46:14,834 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,834 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6723d7b0059e4dd69c96917aebeed90b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/06b112bcf7f947679c71928765a774f5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cd627391288b4450b7da8e7e8d639b57, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6e9f085eebcd42f1a917cb4138f1ea08] 2024-11-14T09:46:14,834 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b2058cddb60499d96b97709aacd6169, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577570281 2024-11-14T09:46:14,835 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 73aa7c509967459bb174a67b3b860fc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577571369 2024-11-14T09:46:14,835 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6723d7b0059e4dd69c96917aebeed90b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577570281 2024-11-14T09:46:14,835 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f493eb50e33144159960288adac56969, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731577572464 2024-11-14T09:46:14,835 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b112bcf7f947679c71928765a774f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577571369 2024-11-14T09:46:14,835 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 692dc8cb64164b16b13c5e8a24cedd03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577573600 2024-11-14T09:46:14,835 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd627391288b4450b7da8e7e8d639b57, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731577572464 2024-11-14T09:46:14,836 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9f085eebcd42f1a917cb4138f1ea08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577573600 2024-11-14T09:46:14,849 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:14,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-14T09:46:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,850 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T09:46:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:14,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:14,858 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:14,859 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e2e57630b4d84b4dbd037d10c469a937 is 50, key is test_row_0/B:col10/1731577574241/Put/seqid=0 2024-11-14T09:46:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T09:46:14,873 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147ed4c79a1c7945bebeb1d369bbb3e7bf_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:14,876 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147ed4c79a1c7945bebeb1d369bbb3e7bf_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:14,876 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147ed4c79a1c7945bebeb1d369bbb3e7bf_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:14,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146bf460ca19644708ada7fae7af23e879_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577574263/Put/seqid=0 2024-11-14T09:46:14,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:14,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742322_1498 (size=12241) 2024-11-14T09:46:14,956 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e2e57630b4d84b4dbd037d10c469a937 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e57630b4d84b4dbd037d10c469a937 2024-11-14T09:46:14,961 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into e2e57630b4d84b4dbd037d10c469a937(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:14,961 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:14,961 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=12, startTime=1731577574830; duration=0sec 2024-11-14T09:46:14,961 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:14,961 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:14,961 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:14,962 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:14,964 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:14,964 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:14,964 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d474996986f34e1ebc97f80f78b801c5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7a84ab006670456e851a299a6328d248, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9bc053e4e13447308a0150badd2cf6c2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ef09a6fadfa241bf94fe46f89e65d25c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=47.0 K 2024-11-14T09:46:14,965 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d474996986f34e1ebc97f80f78b801c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731577570281 2024-11-14T09:46:14,965 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a84ab006670456e851a299a6328d248, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731577571369 2024-11-14T09:46:14,965 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bc053e4e13447308a0150badd2cf6c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731577572464 2024-11-14T09:46:14,965 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ef09a6fadfa241bf94fe46f89e65d25c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577573600 2024-11-14T09:46:14,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742323_1499 (size=4469) 2024-11-14T09:46:14,979 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#420 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:14,979 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5f9d02278ada488b9b55d9ef017b1972 is 175, key is test_row_0/A:col10/1731577574241/Put/seqid=0 2024-11-14T09:46:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577634975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577634981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742324_1500 (size=12204) 2024-11-14T09:46:15,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:15,007 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:15,007 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/abca163916c343d7a2c2596d4aa0ac6d is 50, key is test_row_0/C:col10/1731577574241/Put/seqid=0 2024-11-14T09:46:15,009 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146bf460ca19644708ada7fae7af23e879_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146bf460ca19644708ada7fae7af23e879_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:15,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cdf5a564ee4e43e29ac008911130feb8, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cdf5a564ee4e43e29ac008911130feb8 is 175, key is test_row_0/A:col10/1731577574263/Put/seqid=0 2024-11-14T09:46:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742325_1501 (size=31195) 2024-11-14T09:46:15,064 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5f9d02278ada488b9b55d9ef017b1972 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5f9d02278ada488b9b55d9ef017b1972 2024-11-14T09:46:15,069 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into 5f9d02278ada488b9b55d9ef017b1972(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:15,069 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:15,069 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=12, startTime=1731577574830; duration=0sec 2024-11-14T09:46:15,069 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:15,069 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742326_1502 (size=12241) 2024-11-14T09:46:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577635090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577635090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:15,099 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/abca163916c343d7a2c2596d4aa0ac6d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/abca163916c343d7a2c2596d4aa0ac6d 2024-11-14T09:46:15,104 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into abca163916c343d7a2c2596d4aa0ac6d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:15,104 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:15,104 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=12, startTime=1731577574830; duration=0sec 2024-11-14T09:46:15,104 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:15,104 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:15,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742327_1503 (size=31005) 2024-11-14T09:46:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T09:46:15,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577635300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577635300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:15,511 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cdf5a564ee4e43e29ac008911130feb8 2024-11-14T09:46:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e2e70869c3aa48c3b4f7e70475e1debc is 50, key is test_row_0/B:col10/1731577574263/Put/seqid=0 2024-11-14T09:46:15,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742328_1504 (size=12051) 2024-11-14T09:46:15,561 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e2e70869c3aa48c3b4f7e70475e1debc 2024-11-14T09:46:15,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/fafccfde177349609acf84530fa0d0ba is 50, key is test_row_0/C:col10/1731577574263/Put/seqid=0 2024-11-14T09:46:15,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577635611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:15,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577635611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:15,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742329_1505 (size=12051) 2024-11-14T09:46:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T09:46:16,026 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/fafccfde177349609acf84530fa0d0ba 2024-11-14T09:46:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cdf5a564ee4e43e29ac008911130feb8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cdf5a564ee4e43e29ac008911130feb8 2024-11-14T09:46:16,034 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cdf5a564ee4e43e29ac008911130feb8, entries=150, sequenceid=130, filesize=30.3 K 2024-11-14T09:46:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/e2e70869c3aa48c3b4f7e70475e1debc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e70869c3aa48c3b4f7e70475e1debc 2024-11-14T09:46:16,041 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e70869c3aa48c3b4f7e70475e1debc, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T09:46:16,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/fafccfde177349609acf84530fa0d0ba as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/fafccfde177349609acf84530fa0d0ba 2024-11-14T09:46:16,050 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/fafccfde177349609acf84530fa0d0ba, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T09:46:16,051 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for ee11adb61130645457ff22474228d88e in 1200ms, sequenceid=130, compaction requested=false 2024-11-14T09:46:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-14T09:46:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-14T09:46:16,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-14T09:46:16,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5080 sec 2024-11-14T09:46:16,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.5160 sec 2024-11-14T09:46:16,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T09:46:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577574949, max timestamp=1731577576134Num uniques 859; 2024-11-14T09:46:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577574949, max timestamp=1731577576134Num uniques 859; 2024-11-14T09:46:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577574949, max timestamp=1731577576134Num uniques 859; 2024-11-14T09:46:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:16,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:16,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:16,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:16,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:16,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:16,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:16,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:16,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b533c3a3bb5d4b5097b295dc256b86bb_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577576134/Put/seqid=0 2024-11-14T09:46:16,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577576150, max timestamp=1731577576199Num uniques 859; 2024-11-14T09:46:16,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:16,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577576150, max timestamp=1731577576200Num uniques 859; 2024-11-14T09:46:16,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:16,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577576150, max timestamp=1731577576199Num uniques 859; 2024-11-14T09:46:16,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:16,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742330_1506 (size=14794) 2024-11-14T09:46:16,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731577576295, max timestamp=1731577576314Num uniques 1018; 2024-11-14T09:46:16,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:46:16,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731577576295, max timestamp=1731577576314Num uniques 1018; 2024-11-14T09:46:16,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:46:16,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731577576295, max timestamp=1731577576314Num uniques 1018; 2024-11-14T09:46:16,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T09:46:16,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731577576360, max timestamp=1731577576383Num uniques 1077; 2024-11-14T09:46:16,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731577576360, max timestamp=1731577576383Num uniques 1077; 2024-11-14T09:46:16,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T09:46:16,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T09:46:16,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731577576373, max timestamp=1731577576383Num uniques 1077; 2024-11-14T09:46:16,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-14T09:46:16,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731577576408, max timestamp=1731577576476Num uniques 1086; 2024-11-14T09:46:16,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731577576408, max timestamp=1731577576476Num uniques 1086; 2024-11-14T09:46:16,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T09:46:16,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T09:46:16,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731577576408, max timestamp=1731577576476Num uniques 1036; 2024-11-14T09:46:16,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T09:46:16,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731577576534, max timestamp=1731577576567Num uniques 1045; 2024-11-14T09:46:16,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731577576534, max timestamp=1731577576567Num uniques 1045; 2024-11-14T09:46:16,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T09:46:16,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T09:46:16,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731577576534, max timestamp=1731577576567Num uniques 1045; 2024-11-14T09:46:16,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T09:46:16,623 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:16,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b533c3a3bb5d4b5097b295dc256b86bb_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b533c3a3bb5d4b5097b295dc256b86bb_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:16,630 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/07070dc7d8d74590beaff25a77a6bb1a, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:16,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/07070dc7d8d74590beaff25a77a6bb1a is 175, key is test_row_0/A:col10/1731577576134/Put/seqid=0 2024-11-14T09:46:16,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742331_1507 (size=39749) 2024-11-14T09:46:16,642 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/07070dc7d8d74590beaff25a77a6bb1a 2024-11-14T09:46:16,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731577576597, max timestamp=1731577576650Num uniques 1054; 2024-11-14T09:46:16,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731577576597, max timestamp=1731577576650Num uniques 1054; 2024-11-14T09:46:16,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-14T09:46:16,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-14T09:46:16,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731577576597, max timestamp=1731577576650Num uniques 1054; 2024-11-14T09:46:16,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-14T09:46:16,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/33006ce292604b4f878e19e8d8742d0a is 50, key is test_row_0/B:col10/1731577576134/Put/seqid=0 2024-11-14T09:46:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742332_1508 (size=12151) 2024-11-14T09:46:16,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731577576682, max timestamp=1731577576694Num uniques 1113; 2024-11-14T09:46:16,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-14T09:46:16,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731577576682, max timestamp=1731577576694Num uniques 1113; 2024-11-14T09:46:16,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-14T09:46:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-14T09:46:16,700 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:16,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731577576682, max timestamp=1731577576694Num uniques 1113; 2024-11-14T09:46:16,701 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:16,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-14T09:46:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-14T09:46:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T09:46:16,704 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:16,705 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:16,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:16,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731577576711, max timestamp=1731577576735Num uniques 1022; 2024-11-14T09:46:16,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731577576711, max timestamp=1731577576735Num uniques 1022; 2024-11-14T09:46:16,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-14T09:46:16,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-14T09:46:16,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731577576711, max timestamp=1731577576735Num uniques 1022; 2024-11-14T09:46:16,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-14T09:46:16,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731577576757, max timestamp=1731577576772Num uniques 1131; 2024-11-14T09:46:16,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-14T09:46:16,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731577576757, max timestamp=1731577576772Num uniques 1131; 2024-11-14T09:46:16,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-14T09:46:16,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731577576757, max timestamp=1731577576772Num uniques 1131; 2024-11-14T09:46:16,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-14T09:46:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T09:46:16,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731577576803, max timestamp=1731577576811Num uniques 1040; 2024-11-14T09:46:16,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-14T09:46:16,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731577576803, max timestamp=1731577576811Num uniques 1040; 2024-11-14T09:46:16,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-14T09:46:16,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731577576804, max timestamp=1731577576811Num uniques 1040; 2024-11-14T09:46:16,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-14T09:46:16,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:16,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:16,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:16,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:16,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:16,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731577576851, max timestamp=1731577576869Num uniques 1099; 2024-11-14T09:46:16,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-14T09:46:16,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731577576851, max timestamp=1731577576869Num uniques 1099; 2024-11-14T09:46:16,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-14T09:46:16,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731577576851, max timestamp=1731577576870Num uniques 1049; 2024-11-14T09:46:16,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-14T09:46:16,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731577576905, max timestamp=1731577576914Num uniques 1108; 2024-11-14T09:46:16,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731577576905, max timestamp=1731577576914Num uniques 1108; 2024-11-14T09:46:16,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-14T09:46:16,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-14T09:46:16,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731577576905, max timestamp=1731577576914Num uniques 1108; 2024-11-14T09:46:16,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-14T09:46:16,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731577576938, max timestamp=1731577576952Num uniques 1017; 2024-11-14T09:46:16,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-14T09:46:16,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731577576938, max timestamp=1731577576952Num uniques 1017; 2024-11-14T09:46:16,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731577576938, max timestamp=1731577576952Num uniques 1017; 2024-11-14T09:46:16,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-14T09:46:16,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-14T09:46:17,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731577576984, max timestamp=1731577576995Num uniques 1076; 2024-11-14T09:46:17,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731577576984, max timestamp=1731577576995Num uniques 1076; 2024-11-14T09:46:17,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-14T09:46:17,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-11-14T09:46:17,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731577576984, max timestamp=1731577576995Num uniques 1076; 2024-11-14T09:46:17,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=6870, suffix heap size=139872,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=150 2024-11-14T09:46:17,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:17,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:17,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T09:46:17,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/33006ce292604b4f878e19e8d8742d0a 2024-11-14T09:46:17,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731577577057, max timestamp=1731577577082Num uniques 1035; 2024-11-14T09:46:17,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731577577057, max timestamp=1731577577082Num uniques 1009; 2024-11-14T09:46:17,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8470, suffix heap size=132904,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=185 2024-11-14T09:46:17,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-11-14T09:46:17,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731577577057, max timestamp=1731577577082Num uniques 1035; 2024-11-14T09:46:17,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-11-14T09:46:17,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8cc5e28dda46452c93f0d84d26dbb17d is 50, key is test_row_0/C:col10/1731577576134/Put/seqid=0 2024-11-14T09:46:17,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742333_1509 (size=12151) 2024-11-14T09:46:17,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8cc5e28dda46452c93f0d84d26dbb17d 2024-11-14T09:46:17,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/07070dc7d8d74590beaff25a77a6bb1a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/07070dc7d8d74590beaff25a77a6bb1a 2024-11-14T09:46:17,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/07070dc7d8d74590beaff25a77a6bb1a, entries=200, sequenceid=154, filesize=38.8 K 2024-11-14T09:46:17,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/33006ce292604b4f878e19e8d8742d0a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/33006ce292604b4f878e19e8d8742d0a 2024-11-14T09:46:17,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/33006ce292604b4f878e19e8d8742d0a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T09:46:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8cc5e28dda46452c93f0d84d26dbb17d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8cc5e28dda46452c93f0d84d26dbb17d 2024-11-14T09:46:17,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8cc5e28dda46452c93f0d84d26dbb17d, entries=150, sequenceid=154, filesize=11.9 K 2024-11-14T09:46:17,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=120.76 KB/123660 for ee11adb61130645457ff22474228d88e in 993ms, sequenceid=154, compaction requested=true 2024-11-14T09:46:17,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:17,128 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:17,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:17,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:17,129 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:17,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:17,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:17,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:17,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:17,131 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:17,131 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:17,131 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:17,131 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:17,131 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,131 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,131 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e57630b4d84b4dbd037d10c469a937, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e70869c3aa48c3b4f7e70475e1debc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/33006ce292604b4f878e19e8d8742d0a] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=35.6 K 2024-11-14T09:46:17,131 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5f9d02278ada488b9b55d9ef017b1972, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cdf5a564ee4e43e29ac008911130feb8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/07070dc7d8d74590beaff25a77a6bb1a] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=99.6 K 2024-11-14T09:46:17,131 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,132 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5f9d02278ada488b9b55d9ef017b1972, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cdf5a564ee4e43e29ac008911130feb8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/07070dc7d8d74590beaff25a77a6bb1a] 2024-11-14T09:46:17,132 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e57630b4d84b4dbd037d10c469a937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577573600 2024-11-14T09:46:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:17,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=127.47 KB heapSize=362.63 KB 2024-11-14T09:46:17,136 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e70869c3aa48c3b4f7e70475e1debc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731577574263 2024-11-14T09:46:17,137 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 33006ce292604b4f878e19e8d8742d0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577574972 2024-11-14T09:46:17,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:17,138 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f9d02278ada488b9b55d9ef017b1972, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577573600 2024-11-14T09:46:17,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:17,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:17,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:17,138 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf5a564ee4e43e29ac008911130feb8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731577574263 2024-11-14T09:46:17,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:17,138 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07070dc7d8d74590beaff25a77a6bb1a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577574949 2024-11-14T09:46:17,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,154 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577637143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,155 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/d42d19c664454d2590314a539fb5a70d is 50, key is test_row_0/B:col10/1731577576134/Put/seqid=0 2024-11-14T09:46:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577637150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577637151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577637152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577637154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,181 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:17,190 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a9e00852203f40908af50303066fcf4d_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:17,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140c2b2f5d7926408fb0e80fdea1e3c9cc_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577577111/Put/seqid=0 2024-11-14T09:46:17,192 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a9e00852203f40908af50303066fcf4d_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:17,193 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a9e00852203f40908af50303066fcf4d_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:17,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742334_1510 (size=12493) 2024-11-14T09:46:17,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742335_1511 (size=12454) 2024-11-14T09:46:17,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742336_1512 (size=4469) 2024-11-14T09:46:17,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577637260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577637271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577637271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577637271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577637272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T09:46:17,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577637470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577637480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577637481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577637481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577637481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,597 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/d42d19c664454d2590314a539fb5a70d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/d42d19c664454d2590314a539fb5a70d 2024-11-14T09:46:17,601 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into d42d19c664454d2590314a539fb5a70d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:17,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:17,601 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=13, startTime=1731577577129; duration=0sec 2024-11-14T09:46:17,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:17,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:17,601 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:17,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:17,602 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:17,602 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:17,602 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,602 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/abca163916c343d7a2c2596d4aa0ac6d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/fafccfde177349609acf84530fa0d0ba, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8cc5e28dda46452c93f0d84d26dbb17d] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=35.6 K 2024-11-14T09:46:17,603 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting abca163916c343d7a2c2596d4aa0ac6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577573600 2024-11-14T09:46:17,603 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting fafccfde177349609acf84530fa0d0ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731577574263 2024-11-14T09:46:17,603 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc5e28dda46452c93f0d84d26dbb17d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577574972 2024-11-14T09:46:17,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140c2b2f5d7926408fb0e80fdea1e3c9cc_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140c2b2f5d7926408fb0e80fdea1e3c9cc_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:17,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5986691ce4a34d408d0ad232e08cf542, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:17,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5986691ce4a34d408d0ad232e08cf542 is 175, key is test_row_0/A:col10/1731577577111/Put/seqid=0 2024-11-14T09:46:17,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742337_1513 (size=31255) 2024-11-14T09:46:17,614 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:17,614 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/7562052032f04e3986a8aaada84792a9 is 50, key is test_row_0/C:col10/1731577576134/Put/seqid=0 2024-11-14T09:46:17,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742338_1514 (size=12493) 2024-11-14T09:46:17,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:17,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:17,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,629 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#430 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:17,630 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/69d7c10e09f24cb5a36cd217ca608b0d is 175, key is test_row_0/A:col10/1731577576134/Put/seqid=0 2024-11-14T09:46:17,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742339_1515 (size=31447) 2024-11-14T09:46:17,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577637780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577637791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577637791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577637791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577637791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T09:46:17,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:17,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:17,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:17,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5986691ce4a34d408d0ad232e08cf542 2024-11-14T09:46:18,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/0e8c024735d94fd9844c67c9cb3ddbf9 is 50, key is test_row_0/B:col10/1731577577111/Put/seqid=0 2024-11-14T09:46:18,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742340_1516 (size=12301) 2024-11-14T09:46:18,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/0e8c024735d94fd9844c67c9cb3ddbf9 2024-11-14T09:46:18,021 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/7562052032f04e3986a8aaada84792a9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7562052032f04e3986a8aaada84792a9 2024-11-14T09:46:18,025 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into 7562052032f04e3986a8aaada84792a9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:18,025 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:18,025 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=13, startTime=1731577577130; duration=0sec 2024-11-14T09:46:18,025 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:18,025 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:18,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/4f5dc883a1654bf788fa28e9771e082c is 50, key is test_row_0/C:col10/1731577577111/Put/seqid=0 2024-11-14T09:46:18,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742341_1517 (size=12301) 2024-11-14T09:46:18,040 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/69d7c10e09f24cb5a36cd217ca608b0d as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/69d7c10e09f24cb5a36cd217ca608b0d 2024-11-14T09:46:18,045 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into 69d7c10e09f24cb5a36cd217ca608b0d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:18,045 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:18,045 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=13, startTime=1731577577128; duration=0sec 2024-11-14T09:46:18,045 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:18,045 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:18,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:18,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:18,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577638290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:18,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577638300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577638300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577638300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577638300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:18,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:18,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:18,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/4f5dc883a1654bf788fa28e9771e082c 2024-11-14T09:46:18,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5986691ce4a34d408d0ad232e08cf542 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5986691ce4a34d408d0ad232e08cf542 2024-11-14T09:46:18,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5986691ce4a34d408d0ad232e08cf542, entries=150, sequenceid=431, filesize=30.5 K 2024-11-14T09:46:18,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/0e8c024735d94fd9844c67c9cb3ddbf9 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/0e8c024735d94fd9844c67c9cb3ddbf9 2024-11-14T09:46:18,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/0e8c024735d94fd9844c67c9cb3ddbf9, entries=150, sequenceid=431, filesize=12.0 K 2024-11-14T09:46:18,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/4f5dc883a1654bf788fa28e9771e082c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/4f5dc883a1654bf788fa28e9771e082c 2024-11-14T09:46:18,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/4f5dc883a1654bf788fa28e9771e082c, entries=150, sequenceid=431, filesize=12.0 K 2024-11-14T09:46:18,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~382.50 KB/391680, currentSize=46.96 KB/48090 for ee11adb61130645457ff22474228d88e in 1310ms, sequenceid=431, compaction requested=false 2024-11-14T09:46:18,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:18,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-14T09:46:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:18,543 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:46:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:18,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114990b31e0eb904c0d8eecc0aeb3ce900e_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577577152/Put/seqid=0 2024-11-14T09:46:18,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742342_1518 (size=12454) 2024-11-14T09:46:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T09:46:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:18,954 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114990b31e0eb904c0d8eecc0aeb3ce900e_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114990b31e0eb904c0d8eecc0aeb3ce900e_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/8c06464837b84a8b9e3247cf80627d6f, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/8c06464837b84a8b9e3247cf80627d6f is 175, key is test_row_0/A:col10/1731577577152/Put/seqid=0 2024-11-14T09:46:18,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742343_1519 (size=31255) 2024-11-14T09:46:19,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:19,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577579329, max timestamp=1731577579340Num uniques 859; 2024-11-14T09:46:19,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:46:19,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577579329, max timestamp=1731577579340Num uniques 859; 2024-11-14T09:46:19,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:46:19,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731577579329, max timestamp=1731577579340Num uniques 859; 2024-11-14T09:46:19,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T09:46:19,358 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/8c06464837b84a8b9e3247cf80627d6f 2024-11-14T09:46:19,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577639356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577639357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577639358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aa7b88cedde842fabffcddb92381a240 is 50, key is test_row_0/B:col10/1731577577152/Put/seqid=0 2024-11-14T09:46:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577639361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742344_1520 (size=12301) 2024-11-14T09:46:19,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577639363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577639470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577639471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577639471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577639471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577639481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577639680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577639680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577639680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577639680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577639690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:19,768 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aa7b88cedde842fabffcddb92381a240 2024-11-14T09:46:19,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/9f7a0330c6b64ebd8a761b6d84fcab42 is 50, key is test_row_0/C:col10/1731577577152/Put/seqid=0 2024-11-14T09:46:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742345_1521 (size=12301) 2024-11-14T09:46:19,778 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/9f7a0330c6b64ebd8a761b6d84fcab42 2024-11-14T09:46:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/8c06464837b84a8b9e3247cf80627d6f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/8c06464837b84a8b9e3247cf80627d6f 2024-11-14T09:46:19,786 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/8c06464837b84a8b9e3247cf80627d6f, entries=150, sequenceid=444, filesize=30.5 K 2024-11-14T09:46:19,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aa7b88cedde842fabffcddb92381a240 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aa7b88cedde842fabffcddb92381a240 2024-11-14T09:46:19,789 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aa7b88cedde842fabffcddb92381a240, entries=150, sequenceid=444, filesize=12.0 K 2024-11-14T09:46:19,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/9f7a0330c6b64ebd8a761b6d84fcab42 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9f7a0330c6b64ebd8a761b6d84fcab42 2024-11-14T09:46:19,792 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9f7a0330c6b64ebd8a761b6d84fcab42, entries=150, sequenceid=444, filesize=12.0 K 2024-11-14T09:46:19,793 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ee11adb61130645457ff22474228d88e in 1251ms, sequenceid=444, compaction requested=true 2024-11-14T09:46:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-14T09:46:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-14T09:46:19,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-14T09:46:19,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0890 sec 2024-11-14T09:46:19,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 3.0950 sec 2024-11-14T09:46:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:19,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-14T09:46:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:19,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dc00bc244b064911993453101bd434bf_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577579360/Put/seqid=0 2024-11-14T09:46:20,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742346_1522 (size=14994) 2024-11-14T09:46:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577640000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577640000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577640002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577640002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577640002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577640110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577640120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577640121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577640121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577640320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577640330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577640331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577640331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,403 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:20,406 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dc00bc244b064911993453101bd434bf_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dc00bc244b064911993453101bd434bf_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:20,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/3d172378d56c4755bca623ddb626af21, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:20,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/3d172378d56c4755bca623ddb626af21 is 175, key is test_row_0/A:col10/1731577579360/Put/seqid=0 2024-11-14T09:46:20,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742347_1523 (size=39949) 2024-11-14T09:46:20,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577640520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577640630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577640640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577640640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577640640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:20,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/3d172378d56c4755bca623ddb626af21 2024-11-14T09:46:20,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/a098ba265e7f4eb9986d2cdc5cf6fa74 is 50, key is test_row_0/B:col10/1731577579360/Put/seqid=0 2024-11-14T09:46:20,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742348_1524 (size=12301) 2024-11-14T09:46:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T09:46:20,870 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:20,871 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-14T09:46:20,872 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:46:20,873 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:20,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:46:21,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T09:46:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577641140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:21,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577641150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577641150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577641151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:21,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T09:46:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:46:21,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/a098ba265e7f4eb9986d2cdc5cf6fa74 2024-11-14T09:46:21,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c24fc16b1d30463f95c975c54ca8f2db is 50, key is test_row_0/C:col10/1731577579360/Put/seqid=0 2024-11-14T09:46:21,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742349_1525 (size=12301) 2024-11-14T09:46:21,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T09:46:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T09:46:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:46:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577641530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:21,534 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in ee11adb61130645457ff22474228d88e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:46:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:21,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c24fc16b1d30463f95c975c54ca8f2db 2024-11-14T09:46:21,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T09:46:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:21,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/3d172378d56c4755bca623ddb626af21 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/3d172378d56c4755bca623ddb626af21 2024-11-14T09:46:21,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/3d172378d56c4755bca623ddb626af21, entries=200, sequenceid=479, filesize=39.0 K 2024-11-14T09:46:21,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/a098ba265e7f4eb9986d2cdc5cf6fa74 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/a098ba265e7f4eb9986d2cdc5cf6fa74 2024-11-14T09:46:21,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/a098ba265e7f4eb9986d2cdc5cf6fa74, entries=150, sequenceid=479, filesize=12.0 K 2024-11-14T09:46:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c24fc16b1d30463f95c975c54ca8f2db as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c24fc16b1d30463f95c975c54ca8f2db 2024-11-14T09:46:21,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c24fc16b1d30463f95c975c54ca8f2db, entries=150, sequenceid=479, filesize=12.0 K 2024-11-14T09:46:21,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=46.96 KB/48090 for ee11adb61130645457ff22474228d88e in 1655ms, sequenceid=479, compaction requested=true 2024-11-14T09:46:21,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:21,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:21,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:21,648 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:21,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:21,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:21,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:21,648 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:21,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:21,655 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:21,655 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:21,655 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,655 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/d42d19c664454d2590314a539fb5a70d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/0e8c024735d94fd9844c67c9cb3ddbf9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aa7b88cedde842fabffcddb92381a240, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/a098ba265e7f4eb9986d2cdc5cf6fa74] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=48.2 K 2024-11-14T09:46:21,655 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133906 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:21,655 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:21,655 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,656 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/69d7c10e09f24cb5a36cd217ca608b0d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5986691ce4a34d408d0ad232e08cf542, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/8c06464837b84a8b9e3247cf80627d6f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/3d172378d56c4755bca623ddb626af21] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=130.8 K 2024-11-14T09:46:21,656 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,656 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/69d7c10e09f24cb5a36cd217ca608b0d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5986691ce4a34d408d0ad232e08cf542, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/8c06464837b84a8b9e3247cf80627d6f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/3d172378d56c4755bca623ddb626af21] 2024-11-14T09:46:21,656 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d42d19c664454d2590314a539fb5a70d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577574972 2024-11-14T09:46:21,656 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d7c10e09f24cb5a36cd217ca608b0d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577574972 2024-11-14T09:46:21,656 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e8c024735d94fd9844c67c9cb3ddbf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731577577111 2024-11-14T09:46:21,656 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5986691ce4a34d408d0ad232e08cf542, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731577577111 2024-11-14T09:46:21,656 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c06464837b84a8b9e3247cf80627d6f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731577577139 2024-11-14T09:46:21,656 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting aa7b88cedde842fabffcddb92381a240, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731577577139 2024-11-14T09:46:21,657 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d172378d56c4755bca623ddb626af21, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731577579357 2024-11-14T09:46:21,657 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a098ba265e7f4eb9986d2cdc5cf6fa74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731577579360 2024-11-14T09:46:21,685 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:21,691 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:21,692 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ceb4e5d2276a44f091feff2dd0b06fe3 is 50, key is test_row_0/B:col10/1731577579360/Put/seqid=0 2024-11-14T09:46:21,697 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114bd88af3045054ea7997a88cf70d88e80_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:21,700 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114bd88af3045054ea7997a88cf70d88e80_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:21,700 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bd88af3045054ea7997a88cf70d88e80_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:21,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742350_1526 (size=12779) 2024-11-14T09:46:21,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742351_1527 (size=4469) 2024-11-14T09:46:21,727 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#441 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:21,728 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/c0ab732295e440258eef4ebdee345f8c is 175, key is test_row_0/A:col10/1731577579360/Put/seqid=0 2024-11-14T09:46:21,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742352_1528 (size=31733) 2024-11-14T09:46:21,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-14T09:46:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:21,789 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:46:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114602765018c6b4ea197f482ce0d48f8a7_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577580001/Put/seqid=0 2024-11-14T09:46:21,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742353_1529 (size=12454) 2024-11-14T09:46:21,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:21,804 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114602765018c6b4ea197f482ce0d48f8a7_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114602765018c6b4ea197f482ce0d48f8a7_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:21,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/82303ae2c2df44d9ab6410dbe6c67697, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:21,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/82303ae2c2df44d9ab6410dbe6c67697 is 175, key is test_row_0/A:col10/1731577580001/Put/seqid=0 2024-11-14T09:46:21,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742354_1530 (size=31255) 2024-11-14T09:46:21,824 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=489, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/82303ae2c2df44d9ab6410dbe6c67697 2024-11-14T09:46:21,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aec83e3f2ff243b3ab1dc8f71eb42ba8 is 50, key is test_row_0/B:col10/1731577580001/Put/seqid=0 2024-11-14T09:46:21,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742355_1531 (size=12301) 2024-11-14T09:46:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:46:22,118 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ceb4e5d2276a44f091feff2dd0b06fe3 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb4e5d2276a44f091feff2dd0b06fe3 2024-11-14T09:46:22,126 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into ceb4e5d2276a44f091feff2dd0b06fe3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:22,126 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:22,126 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=12, startTime=1731577581648; duration=0sec 2024-11-14T09:46:22,126 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:22,126 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:22,126 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:22,128 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:22,128 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:22,128 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:22,128 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7562052032f04e3986a8aaada84792a9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/4f5dc883a1654bf788fa28e9771e082c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9f7a0330c6b64ebd8a761b6d84fcab42, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c24fc16b1d30463f95c975c54ca8f2db] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=48.2 K 2024-11-14T09:46:22,128 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 7562052032f04e3986a8aaada84792a9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731577574972 2024-11-14T09:46:22,128 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f5dc883a1654bf788fa28e9771e082c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731577577111 2024-11-14T09:46:22,128 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7a0330c6b64ebd8a761b6d84fcab42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731577577139 2024-11-14T09:46:22,129 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c24fc16b1d30463f95c975c54ca8f2db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731577579360 2024-11-14T09:46:22,135 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:22,136 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/ba04c75d547c4568add173b744da6fad is 50, key is test_row_0/C:col10/1731577579360/Put/seqid=0 2024-11-14T09:46:22,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742356_1532 (size=12779) 2024-11-14T09:46:22,145 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/c0ab732295e440258eef4ebdee345f8c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c0ab732295e440258eef4ebdee345f8c 2024-11-14T09:46:22,149 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into c0ab732295e440258eef4ebdee345f8c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:22,149 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:22,149 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=12, startTime=1731577581648; duration=0sec 2024-11-14T09:46:22,149 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:22,149 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:22,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:22,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577582176, max timestamp=1731577582186Num uniques 859; 2024-11-14T09:46:22,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:22,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577582176, max timestamp=1731577582186Num uniques 859; 2024-11-14T09:46:22,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577582176, max timestamp=1731577582186Num uniques 859; 2024-11-14T09:46:22,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:22,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577642206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577642207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577642209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577642211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,270 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aec83e3f2ff243b3ab1dc8f71eb42ba8 2024-11-14T09:46:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c567080dd0214584b30b5c2ae3f7a5df is 50, key is test_row_0/C:col10/1731577580001/Put/seqid=0 2024-11-14T09:46:22,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742357_1533 (size=12301) 2024-11-14T09:46:22,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577642320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577642320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577642320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577642320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577642530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577642530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577642530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577642530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,543 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/ba04c75d547c4568add173b744da6fad as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ba04c75d547c4568add173b744da6fad 2024-11-14T09:46:22,546 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into ba04c75d547c4568add173b744da6fad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:22,546 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:22,547 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=12, startTime=1731577581648; duration=0sec 2024-11-14T09:46:22,547 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:22,547 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:22,681 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c567080dd0214584b30b5c2ae3f7a5df 2024-11-14T09:46:22,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/82303ae2c2df44d9ab6410dbe6c67697 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/82303ae2c2df44d9ab6410dbe6c67697 2024-11-14T09:46:22,686 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/82303ae2c2df44d9ab6410dbe6c67697, entries=150, sequenceid=489, filesize=30.5 K 2024-11-14T09:46:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aec83e3f2ff243b3ab1dc8f71eb42ba8 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aec83e3f2ff243b3ab1dc8f71eb42ba8 2024-11-14T09:46:22,690 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aec83e3f2ff243b3ab1dc8f71eb42ba8, entries=150, sequenceid=489, filesize=12.0 K 2024-11-14T09:46:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/c567080dd0214584b30b5c2ae3f7a5df as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c567080dd0214584b30b5c2ae3f7a5df 2024-11-14T09:46:22,692 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c567080dd0214584b30b5c2ae3f7a5df, entries=150, sequenceid=489, filesize=12.0 K 2024-11-14T09:46:22,693 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ee11adb61130645457ff22474228d88e in 904ms, sequenceid=489, compaction requested=false 2024-11-14T09:46:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-14T09:46:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-14T09:46:22,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-14T09:46:22,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8210 sec 2024-11-14T09:46:22,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.8250 sec 2024-11-14T09:46:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:22,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-14T09:46:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:46:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:46:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:46:22,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb9aa90c103840c9a24d175772eae5f5_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577582844/Put/seqid=0 2024-11-14T09:46:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577642847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577642851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577642851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577642852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742358_1534 (size=17534) 2024-11-14T09:46:22,858 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:22,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb9aa90c103840c9a24d175772eae5f5_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb9aa90c103840c9a24d175772eae5f5_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:22,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/c89c46bdd26c4d4393bcb9e35ef5d57c, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:22,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/c89c46bdd26c4d4393bcb9e35ef5d57c is 175, key is test_row_0/A:col10/1731577582844/Put/seqid=0 2024-11-14T09:46:22,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742359_1535 (size=48639) 2024-11-14T09:46:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577642960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577642960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577642960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577642960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:46:23,021 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:23,021 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-14T09:46:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:46:23,023 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:23,024 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:23,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:46:23,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577643170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577643170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577643170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577643170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:23,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:23,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=533, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/c89c46bdd26c4d4393bcb9e35ef5d57c 2024-11-14T09:46:23,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/48e9967f37844a22a6818a9bf0c813ca is 50, key is test_row_0/B:col10/1731577582844/Put/seqid=0 2024-11-14T09:46:23,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742360_1536 (size=12301) 2024-11-14T09:46:23,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:23,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:23,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:46:23,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577643480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577643480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577643480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577643480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577643550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,553 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in ee11adb61130645457ff22474228d88e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:46:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:46:23,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/48e9967f37844a22a6818a9bf0c813ca 2024-11-14T09:46:23,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/929a4883b61645fabb6370e6e71a07dc is 50, key is test_row_0/C:col10/1731577582844/Put/seqid=0 2024-11-14T09:46:23,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742361_1537 (size=12301) 2024-11-14T09:46:23,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:23,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577643990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577643990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577643990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577643990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:24,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:24,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:24,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/929a4883b61645fabb6370e6e71a07dc 2024-11-14T09:46:24,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/c89c46bdd26c4d4393bcb9e35ef5d57c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c89c46bdd26c4d4393bcb9e35ef5d57c 2024-11-14T09:46:24,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c89c46bdd26c4d4393bcb9e35ef5d57c, entries=250, sequenceid=533, filesize=47.5 K 2024-11-14T09:46:24,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/48e9967f37844a22a6818a9bf0c813ca as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/48e9967f37844a22a6818a9bf0c813ca 2024-11-14T09:46:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/48e9967f37844a22a6818a9bf0c813ca, entries=150, sequenceid=533, filesize=12.0 K 2024-11-14T09:46:24,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/929a4883b61645fabb6370e6e71a07dc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/929a4883b61645fabb6370e6e71a07dc 2024-11-14T09:46:24,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/929a4883b61645fabb6370e6e71a07dc, entries=150, sequenceid=533, filesize=12.0 K 2024-11-14T09:46:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~465.33 KB/476496, currentSize=20.13 KB/20610 for ee11adb61130645457ff22474228d88e in 1279ms, sequenceid=533, compaction requested=true 2024-11-14T09:46:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:24,124 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:46:24,125 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,126 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:24,126 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:24,126 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:24,126 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c0ab732295e440258eef4ebdee345f8c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/82303ae2c2df44d9ab6410dbe6c67697, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c89c46bdd26c4d4393bcb9e35ef5d57c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=109.0 K 2024-11-14T09:46:24,126 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:24,126 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c0ab732295e440258eef4ebdee345f8c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/82303ae2c2df44d9ab6410dbe6c67697, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c89c46bdd26c4d4393bcb9e35ef5d57c] 2024-11-14T09:46:24,127 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:24,127 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:24,127 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,127 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb4e5d2276a44f091feff2dd0b06fe3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aec83e3f2ff243b3ab1dc8f71eb42ba8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/48e9967f37844a22a6818a9bf0c813ca] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=36.5 K 2024-11-14T09:46:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,127 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb4e5d2276a44f091feff2dd0b06fe3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731577579360 2024-11-14T09:46:24,127 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0ab732295e440258eef4ebdee345f8c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731577579360 2024-11-14T09:46:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,128 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82303ae2c2df44d9ab6410dbe6c67697, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731577579995 2024-11-14T09:46:24,128 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting c89c46bdd26c4d4393bcb9e35ef5d57c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731577582204 2024-11-14T09:46:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,129 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting aec83e3f2ff243b3ab1dc8f71eb42ba8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731577579995 2024-11-14T09:46:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,130 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 48e9967f37844a22a6818a9bf0c813ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731577582204 2024-11-14T09:46:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,151 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,151 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ccea419231be4a868329404a6e4e5b6e is 50, key is test_row_0/B:col10/1731577582844/Put/seqid=0 2024-11-14T09:46:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:46:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,162 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,166 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a106d4a4843141a19b9591da8ed7de92_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742362_1538 (size=12881) 2024-11-14T09:46:24,170 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a106d4a4843141a19b9591da8ed7de92_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:24,170 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a106d4a4843141a19b9591da8ed7de92_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,176 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ccea419231be4a868329404a6e4e5b6e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ccea419231be4a868329404a6e4e5b6e 2024-11-14T09:46:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,182 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into ccea419231be4a868329404a6e4e5b6e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:24,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,182 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=13, startTime=1731577584124; duration=0sec 2024-11-14T09:46:24,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:24,182 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:24,183 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,183 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,183 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:24,184 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:24,184 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ba04c75d547c4568add173b744da6fad, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c567080dd0214584b30b5c2ae3f7a5df, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/929a4883b61645fabb6370e6e71a07dc] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=36.5 K 2024-11-14T09:46:24,184 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ba04c75d547c4568add173b744da6fad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731577579360 2024-11-14T09:46:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,184 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting c567080dd0214584b30b5c2ae3f7a5df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731577579995 2024-11-14T09:46:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,185 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 929a4883b61645fabb6370e6e71a07dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731577582204 2024-11-14T09:46:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742363_1539 (size=4469) 2024-11-14T09:46:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,215 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,216 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/d03748bebe464c98bcba43a0cad600ac is 50, key is test_row_0/C:col10/1731577582844/Put/seqid=0 2024-11-14T09:46:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T09:46:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,246 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T09:46:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742364_1540 (size=12881) 2024-11-14T09:46:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,253 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/d03748bebe464c98bcba43a0cad600ac as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d03748bebe464c98bcba43a0cad600ac 2024-11-14T09:46:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,259 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into d03748bebe464c98bcba43a0cad600ac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:24,259 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:24,259 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=13, startTime=1731577584124; duration=0sec 2024-11-14T09:46:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,259 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:24,259 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141eb13d8a38314dbf988533a9048e92c1_ee11adb61130645457ff22474228d88e is 50, key is test_row_1/A:col10/1731577582851/Put/seqid=0 2024-11-14T09:46:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742365_1541 (size=9914) 2024-11-14T09:46:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,295 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141eb13d8a38314dbf988533a9048e92c1_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141eb13d8a38314dbf988533a9048e92c1_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b06842588cf743ef99e1f8fb90f0fa4c, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b06842588cf743ef99e1f8fb90f0fa4c is 175, key is test_row_1/A:col10/1731577582851/Put/seqid=0 2024-11-14T09:46:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742366_1542 (size=22561) 2024-11-14T09:46:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,307 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b06842588cf743ef99e1f8fb90f0fa4c 2024-11-14T09:46:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f8b18740b58e481a9486cdc421ecce04 is 50, key is test_row_1/B:col10/1731577582851/Put/seqid=0 2024-11-14T09:46:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742367_1543 (size=9857) 2024-11-14T09:46:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,614 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#451 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,614 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/26d6216a95684fb49fcbdd6431bec0ce is 175, key is test_row_0/A:col10/1731577582844/Put/seqid=0 2024-11-14T09:46:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742368_1544 (size=31835) 2024-11-14T09:46:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,765 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f8b18740b58e481a9486cdc421ecce04 2024-11-14T09:46:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/96375d46d9314966b75bcb431f4b802b is 50, key is test_row_1/C:col10/1731577582851/Put/seqid=0 2024-11-14T09:46:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742369_1545 (size=9857) 2024-11-14T09:46:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:25,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731577585002, max timestamp=1731577585041Num uniques 859; 2024-11-14T09:46:25,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731577585002, max timestamp=1731577585041Num uniques 859; 2024-11-14T09:46:25,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T09:46:25,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T09:46:25,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577585018, max timestamp=1731577585041Num uniques 859; 2024-11-14T09:46:25,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:46:25,062 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/26d6216a95684fb49fcbdd6431bec0ce as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/26d6216a95684fb49fcbdd6431bec0ce 2024-11-14T09:46:25,069 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into 26d6216a95684fb49fcbdd6431bec0ce(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:25,069 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:25,069 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=13, startTime=1731577584124; duration=0sec 2024-11-14T09:46:25,069 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:25,069 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577645069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577645069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577645072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577645069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:46:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577645181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577645190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577645190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577645190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,201 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/96375d46d9314966b75bcb431f4b802b 2024-11-14T09:46:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b06842588cf743ef99e1f8fb90f0fa4c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b06842588cf743ef99e1f8fb90f0fa4c 2024-11-14T09:46:25,210 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b06842588cf743ef99e1f8fb90f0fa4c, entries=100, sequenceid=540, filesize=22.0 K 2024-11-14T09:46:25,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f8b18740b58e481a9486cdc421ecce04 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f8b18740b58e481a9486cdc421ecce04 2024-11-14T09:46:25,216 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f8b18740b58e481a9486cdc421ecce04, entries=100, sequenceid=540, filesize=9.6 K 2024-11-14T09:46:25,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/96375d46d9314966b75bcb431f4b802b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/96375d46d9314966b75bcb431f4b802b 2024-11-14T09:46:25,224 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/96375d46d9314966b75bcb431f4b802b, entries=100, sequenceid=540, filesize=9.6 K 2024-11-14T09:46:25,224 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=172.20 KB/176330 for ee11adb61130645457ff22474228d88e in 979ms, sequenceid=540, compaction requested=false 2024-11-14T09:46:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-14T09:46:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-14T09:46:25,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T09:46:25,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2020 sec 2024-11-14T09:46:25,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.2100 sec 2024-11-14T09:46:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:25,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=178.91 KB heapSize=472.48 KB 2024-11-14T09:46:25,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:25,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:25,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:25,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:25,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:25,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:25,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577645400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577645401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577645401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577645405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145aacd138c1fc4e0cafa2cbb8319c64cf_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577585065/Put/seqid=0 2024-11-14T09:46:25,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742370_1546 (size=14994) 2024-11-14T09:46:25,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145aacd138c1fc4e0cafa2cbb8319c64cf_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145aacd138c1fc4e0cafa2cbb8319c64cf_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:25,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cfcfda0465bc4146a503188d9ce68c58, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:25,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cfcfda0465bc4146a503188d9ce68c58 is 175, key is test_row_0/A:col10/1731577585065/Put/seqid=0 2024-11-14T09:46:25,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742371_1547 (size=39949) 2024-11-14T09:46:25,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=576, memsize=69.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cfcfda0465bc4146a503188d9ce68c58 2024-11-14T09:46:25,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/08dd143226d945738e6a7b7ecce7d57c is 50, key is test_row_0/B:col10/1731577585065/Put/seqid=0 2024-11-14T09:46:25,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577645520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742372_1548 (size=12301) 2024-11-14T09:46:25,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/08dd143226d945738e6a7b7ecce7d57c 2024-11-14T09:46:25,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/b6b2294a010543b29064b6c8a274e12b is 50, key is test_row_0/C:col10/1731577585065/Put/seqid=0 2024-11-14T09:46:25,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742373_1549 (size=12301) 2024-11-14T09:46:25,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577645710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577645711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577645711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577645730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:25,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/b6b2294a010543b29064b6c8a274e12b 2024-11-14T09:46:25,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/cfcfda0465bc4146a503188d9ce68c58 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cfcfda0465bc4146a503188d9ce68c58 2024-11-14T09:46:25,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cfcfda0465bc4146a503188d9ce68c58, entries=200, sequenceid=576, filesize=39.0 K 2024-11-14T09:46:25,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/08dd143226d945738e6a7b7ecce7d57c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/08dd143226d945738e6a7b7ecce7d57c 2024-11-14T09:46:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/08dd143226d945738e6a7b7ecce7d57c, entries=150, sequenceid=576, filesize=12.0 K 2024-11-14T09:46:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/b6b2294a010543b29064b6c8a274e12b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/b6b2294a010543b29064b6c8a274e12b 2024-11-14T09:46:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/b6b2294a010543b29064b6c8a274e12b, entries=150, sequenceid=576, filesize=12.0 K 2024-11-14T09:46:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~178.91 KB/183200, heapSize ~472.44 KB/483776, currentSize=13.42 KB/13740 for ee11adb61130645457ff22474228d88e in 605ms, sequenceid=576, compaction requested=true 2024-11-14T09:46:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,999 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:25,999 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,001 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:26,001 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:26,001 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,002 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/26d6216a95684fb49fcbdd6431bec0ce, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b06842588cf743ef99e1f8fb90f0fa4c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cfcfda0465bc4146a503188d9ce68c58] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=92.1 K 2024-11-14T09:46:26,002 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:26,002 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/26d6216a95684fb49fcbdd6431bec0ce, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b06842588cf743ef99e1f8fb90f0fa4c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cfcfda0465bc4146a503188d9ce68c58] 2024-11-14T09:46:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,002 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26d6216a95684fb49fcbdd6431bec0ce, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731577582204 2024-11-14T09:46:26,003 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06842588cf743ef99e1f8fb90f0fa4c, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731577582850 2024-11-14T09:46:26,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:26,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:26,003 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:26,003 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ccea419231be4a868329404a6e4e5b6e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f8b18740b58e481a9486cdc421ecce04, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/08dd143226d945738e6a7b7ecce7d57c] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=34.2 K 2024-11-14T09:46:26,003 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfcfda0465bc4146a503188d9ce68c58, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731577585065 2024-11-14T09:46:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,003 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting ccea419231be4a868329404a6e4e5b6e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731577582204 2024-11-14T09:46:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,004 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b18740b58e481a9486cdc421ecce04, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731577582850 2024-11-14T09:46:26,004 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 08dd143226d945738e6a7b7ecce7d57c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731577585065 2024-11-14T09:46:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,020 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,030 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,031 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/6150f30d57534d84937a732964afac31 is 50, key is test_row_0/B:col10/1731577585065/Put/seqid=0 2024-11-14T09:46:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,039 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411141bc8c441779e4df880157b7a2c8af31e_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,041 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411141bc8c441779e4df880157b7a2c8af31e_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,041 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141bc8c441779e4df880157b7a2c8af31e_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:26,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:46:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:26,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742374_1550 (size=12983) 2024-11-14T09:46:26,080 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/6150f30d57534d84937a732964afac31 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/6150f30d57534d84937a732964afac31 2024-11-14T09:46:26,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742375_1551 (size=4469) 2024-11-14T09:46:26,085 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into 6150f30d57534d84937a732964afac31(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:26,085 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:26,085 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=13, startTime=1731577585999; duration=0sec 2024-11-14T09:46:26,085 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:26,085 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:26,085 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:26,087 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:26,087 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:26,088 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:26,088 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d03748bebe464c98bcba43a0cad600ac, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/96375d46d9314966b75bcb431f4b802b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/b6b2294a010543b29064b6c8a274e12b] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=34.2 K 2024-11-14T09:46:26,088 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting d03748bebe464c98bcba43a0cad600ac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731577582204 2024-11-14T09:46:26,088 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 96375d46d9314966b75bcb431f4b802b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731577582850 2024-11-14T09:46:26,088 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b2294a010543b29064b6c8a274e12b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731577585065 2024-11-14T09:46:26,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b6f240d374014d7cab991d40321a6bd8_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577586069/Put/seqid=0 2024-11-14T09:46:26,102 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:26,102 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/a913f2bab53e4526ae9c6881d0f66c1e is 50, key is test_row_0/C:col10/1731577585065/Put/seqid=0 2024-11-14T09:46:26,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742376_1552 (size=14994) 2024-11-14T09:46:26,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742377_1553 (size=12983) 2024-11-14T09:46:26,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b6f240d374014d7cab991d40321a6bd8_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b6f240d374014d7cab991d40321a6bd8_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:26,135 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dd285bf639b5482aace3d6ae8d512101, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:26,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dd285bf639b5482aace3d6ae8d512101 is 175, key is test_row_0/A:col10/1731577586069/Put/seqid=0 2024-11-14T09:46:26,140 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/a913f2bab53e4526ae9c6881d0f66c1e as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/a913f2bab53e4526ae9c6881d0f66c1e 2024-11-14T09:46:26,147 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into a913f2bab53e4526ae9c6881d0f66c1e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:26,147 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:26,147 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=13, startTime=1731577586013; duration=0sec 2024-11-14T09:46:26,147 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:26,147 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:26,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742378_1554 (size=39949) 2024-11-14T09:46:26,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=586, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dd285bf639b5482aace3d6ae8d512101 2024-11-14T09:46:26,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/b5f1773eadcd455b883f1a30239d6364 is 50, key is test_row_0/B:col10/1731577586069/Put/seqid=0 2024-11-14T09:46:26,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742379_1555 (size=12301) 2024-11-14T09:46:26,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/b5f1773eadcd455b883f1a30239d6364 2024-11-14T09:46:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577586148, max timestamp=1731577586175Num uniques 859; 2024-11-14T09:46:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577586148, max timestamp=1731577586175Num uniques 859; 2024-11-14T09:46:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731577586148, max timestamp=1731577586175Num uniques 859; 2024-11-14T09:46:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:26,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T09:46:26,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8c7169d5042e48f6ae4f9f4c86aaab38 is 50, key is test_row_0/C:col10/1731577586069/Put/seqid=0 2024-11-14T09:46:26,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742380_1556 (size=12301) 2024-11-14T09:46:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577646232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577646233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577646234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577646239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577646350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577646351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577646351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577646351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,484 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#459 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:26,485 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5b3153e93e4a48e9aff14f59e4d6affb is 175, key is test_row_0/A:col10/1731577585065/Put/seqid=0 2024-11-14T09:46:26,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742381_1557 (size=31937) 2024-11-14T09:46:26,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577646560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577646561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577646561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577646561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8c7169d5042e48f6ae4f9f4c86aaab38 2024-11-14T09:46:26,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dd285bf639b5482aace3d6ae8d512101 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dd285bf639b5482aace3d6ae8d512101 2024-11-14T09:46:26,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dd285bf639b5482aace3d6ae8d512101, entries=200, sequenceid=586, filesize=39.0 K 2024-11-14T09:46:26,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/b5f1773eadcd455b883f1a30239d6364 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/b5f1773eadcd455b883f1a30239d6364 2024-11-14T09:46:26,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/b5f1773eadcd455b883f1a30239d6364, entries=150, sequenceid=586, filesize=12.0 K 2024-11-14T09:46:26,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/8c7169d5042e48f6ae4f9f4c86aaab38 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8c7169d5042e48f6ae4f9f4c86aaab38 2024-11-14T09:46:26,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8c7169d5042e48f6ae4f9f4c86aaab38, entries=150, sequenceid=586, filesize=12.0 K 2024-11-14T09:46:26,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ee11adb61130645457ff22474228d88e in 545ms, sequenceid=586, compaction requested=false 2024-11-14T09:46:26,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:26,643 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:46:26,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-14T09:46:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:46:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:46:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T09:46:26,892 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/5b3153e93e4a48e9aff14f59e4d6affb as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5b3153e93e4a48e9aff14f59e4d6affb 2024-11-14T09:46:26,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145e97588a96994174baca4766eb7d20a3_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577586238/Put/seqid=0 2024-11-14T09:46:26,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742382_1558 (size=12454) 2024-11-14T09:46:26,910 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:26,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145e97588a96994174baca4766eb7d20a3_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145e97588a96994174baca4766eb7d20a3_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:26,914 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6ed9aefed7c74c77abcb562641d8e72f, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:26,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6ed9aefed7c74c77abcb562641d8e72f is 175, key is test_row_0/A:col10/1731577586238/Put/seqid=0 2024-11-14T09:46:26,925 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into 5b3153e93e4a48e9aff14f59e4d6affb(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:26,925 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:26,925 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=13, startTime=1731577585999; duration=0sec 2024-11-14T09:46:26,925 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:26,926 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:26,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577646879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742383_1559 (size=31255) 2024-11-14T09:46:26,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=627, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6ed9aefed7c74c77abcb562641d8e72f 2024-11-14T09:46:26,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577646926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577646926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577646926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:26,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/cbfec3ba3db74edbbb5a3082b2d452c6 is 50, key is test_row_0/B:col10/1731577586238/Put/seqid=0 2024-11-14T09:46:26,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742384_1560 (size=12301) 2024-11-14T09:46:26,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/cbfec3ba3db74edbbb5a3082b2d452c6 2024-11-14T09:46:26,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/bee97cc1445245afb684e88f2964b03a is 50, key is test_row_0/C:col10/1731577586238/Put/seqid=0 2024-11-14T09:46:27,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742385_1561 (size=12301) 2024-11-14T09:46:27,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/bee97cc1445245afb684e88f2964b03a 2024-11-14T09:46:27,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/6ed9aefed7c74c77abcb562641d8e72f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6ed9aefed7c74c77abcb562641d8e72f 2024-11-14T09:46:27,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6ed9aefed7c74c77abcb562641d8e72f, entries=150, sequenceid=627, filesize=30.5 K 2024-11-14T09:46:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/cbfec3ba3db74edbbb5a3082b2d452c6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/cbfec3ba3db74edbbb5a3082b2d452c6 2024-11-14T09:46:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/cbfec3ba3db74edbbb5a3082b2d452c6, entries=150, sequenceid=627, filesize=12.0 K 2024-11-14T09:46:27,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/bee97cc1445245afb684e88f2964b03a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/bee97cc1445245afb684e88f2964b03a 2024-11-14T09:46:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/bee97cc1445245afb684e88f2964b03a, entries=150, sequenceid=627, filesize=12.0 K 2024-11-14T09:46:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for ee11adb61130645457ff22474228d88e in 152ms, sequenceid=627, compaction requested=true 2024-11-14T09:46:27,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:27,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,027 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,028 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:27,029 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,029 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,029 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,029 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5b3153e93e4a48e9aff14f59e4d6affb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dd285bf639b5482aace3d6ae8d512101, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6ed9aefed7c74c77abcb562641d8e72f] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=100.7 K 2024-11-14T09:46:27,029 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,029 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5b3153e93e4a48e9aff14f59e4d6affb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dd285bf639b5482aace3d6ae8d512101, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6ed9aefed7c74c77abcb562641d8e72f] 2024-11-14T09:46:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,029 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:27,029 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:27,029 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,029 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/6150f30d57534d84937a732964afac31, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/b5f1773eadcd455b883f1a30239d6364, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/cbfec3ba3db74edbbb5a3082b2d452c6] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=36.7 K 2024-11-14T09:46:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,030 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b3153e93e4a48e9aff14f59e4d6affb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731577585065 2024-11-14T09:46:27,030 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6150f30d57534d84937a732964afac31, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731577585065 2024-11-14T09:46:27,030 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting dd285bf639b5482aace3d6ae8d512101, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731577586055 2024-11-14T09:46:27,030 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5f1773eadcd455b883f1a30239d6364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731577586055 2024-11-14T09:46:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,030 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbfec3ba3db74edbbb5a3082b2d452c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731577586232 2024-11-14T09:46:27,030 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed9aefed7c74c77abcb562641d8e72f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731577586232 2024-11-14T09:46:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,038 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,038 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f7ad23c0c80d45e09592d5661fc6b0b4 is 50, key is test_row_0/B:col10/1731577586238/Put/seqid=0 2024-11-14T09:46:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,040 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742386_1562 (size=13085) 2024-11-14T09:46:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,045 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111469211a32e795434183a821d6e3c0e63f_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,047 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111469211a32e795434183a821d6e3c0e63f_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:27,047 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469211a32e795434183a821d6e3c0e63f_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742387_1563 (size=4469) 2024-11-14T09:46:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T09:46:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,054 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#469 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:27,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:27,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:27,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:27,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:27,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:27,054 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/52d95a9643f24711b5642720749d286a is 175, key is test_row_0/A:col10/1731577586238/Put/seqid=0 2024-11-14T09:46:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742388_1564 (size=32039) 2024-11-14T09:46:27,064 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/52d95a9643f24711b5642720749d286a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/52d95a9643f24711b5642720749d286a 2024-11-14T09:46:27,069 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into 52d95a9643f24711b5642720749d286a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:27,069 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:27,069 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=13, startTime=1731577587027; duration=0sec 2024-11-14T09:46:27,069 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:27,069 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:27,069 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:46:27,071 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:46:27,071 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:27,071 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,072 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/a913f2bab53e4526ae9c6881d0f66c1e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8c7169d5042e48f6ae4f9f4c86aaab38, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/bee97cc1445245afb684e88f2964b03a] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=36.7 K 2024-11-14T09:46:27,072 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting a913f2bab53e4526ae9c6881d0f66c1e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731577585065 2024-11-14T09:46:27,072 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7169d5042e48f6ae4f9f4c86aaab38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731577586055 2024-11-14T09:46:27,073 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting bee97cc1445245afb684e88f2964b03a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731577586232 2024-11-14T09:46:27,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148c51386c78dd48978026d5f4662f7432_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577587052/Put/seqid=0 2024-11-14T09:46:27,097 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:27,098 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/37f961a8feb1455e998ffd02a0857068 is 50, key is test_row_0/C:col10/1731577586238/Put/seqid=0 2024-11-14T09:46:27,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742389_1565 (size=13085) 2024-11-14T09:46:27,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742390_1566 (size=20074) 2024-11-14T09:46:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577587137, max timestamp=1731577587149Num uniques 859; 2024-11-14T09:46:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577587137, max timestamp=1731577587149Num uniques 859; 2024-11-14T09:46:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:46:27,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:46:27,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731577587137, max timestamp=1731577587149Num uniques 859; 2024-11-14T09:46:27,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/defc576eb6b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T09:46:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T09:46:27,191 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:27,192 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:46:27,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577647185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577647187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-14T09:46:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:46:27,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:46:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577647191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,197 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:46:27,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:46:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577647191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:46:27,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577647300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577647300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577647301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577647301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T09:46:27,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:27,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:27,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:27,450 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/f7ad23c0c80d45e09592d5661fc6b0b4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f7ad23c0c80d45e09592d5661fc6b0b4 2024-11-14T09:46:27,455 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into f7ad23c0c80d45e09592d5661fc6b0b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:27,455 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:27,455 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=13, startTime=1731577587028; duration=0sec 2024-11-14T09:46:27,455 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:27,455 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:27,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T09:46:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:46:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:46:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577647510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577647510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577647510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577647510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,544 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:27,549 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/37f961a8feb1455e998ffd02a0857068 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/37f961a8feb1455e998ffd02a0857068 2024-11-14T09:46:27,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148c51386c78dd48978026d5f4662f7432_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148c51386c78dd48978026d5f4662f7432_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:27,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dcc24aafbb104e70aed4bce128289489, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:27,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dcc24aafbb104e70aed4bce128289489 is 175, key is test_row_0/A:col10/1731577587052/Put/seqid=0 2024-11-14T09:46:27,556 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into 37f961a8feb1455e998ffd02a0857068(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:27,556 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:27,556 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=13, startTime=1731577587028; duration=0sec 2024-11-14T09:46:27,556 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:27,556 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:27,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742391_1567 (size=57333) 2024-11-14T09:46:27,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=640, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dcc24aafbb104e70aed4bce128289489 2024-11-14T09:46:27,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aac736d701944d53a85baac39f47170f is 50, key is test_row_0/B:col10/1731577587052/Put/seqid=0 2024-11-14T09:46:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731577647590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,595 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,42511,1731577408298 for 'test_row_0' in ee11adb61130645457ff22474228d88e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:46:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742392_1568 (size=12301) 2024-11-14T09:46:27,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aac736d701944d53a85baac39f47170f 2024-11-14T09:46:27,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/76cfab180a934877b6d03eb144af527b is 50, key is test_row_0/C:col10/1731577587052/Put/seqid=0 2024-11-14T09:46:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742393_1569 (size=12301) 2024-11-14T09:46:27,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/76cfab180a934877b6d03eb144af527b 2024-11-14T09:46:27,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/dcc24aafbb104e70aed4bce128289489 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dcc24aafbb104e70aed4bce128289489 2024-11-14T09:46:27,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dcc24aafbb104e70aed4bce128289489, entries=300, sequenceid=640, filesize=56.0 K 2024-11-14T09:46:27,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/aac736d701944d53a85baac39f47170f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aac736d701944d53a85baac39f47170f 2024-11-14T09:46:27,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aac736d701944d53a85baac39f47170f, entries=150, sequenceid=640, filesize=12.0 K 2024-11-14T09:46:27,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/76cfab180a934877b6d03eb144af527b as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/76cfab180a934877b6d03eb144af527b 2024-11-14T09:46:27,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/76cfab180a934877b6d03eb144af527b, entries=150, sequenceid=640, filesize=12.0 K 2024-11-14T09:46:27,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for ee11adb61130645457ff22474228d88e in 618ms, sequenceid=640, compaction requested=false 2024-11-14T09:46:27,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:27,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42511 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T09:46:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:27,695 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T09:46:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T09:46:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b80bc4d278c647ef94c28b195a2d988e_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577587189/Put/seqid=0 2024-11-14T09:46:27,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742394_1570 (size=12454) 2024-11-14T09:46:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:46:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. as already flushing 2024-11-14T09:46:27,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577647832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577647833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577647834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577647834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577647940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577647951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577647951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:27,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577647951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:28,131 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b80bc4d278c647ef94c28b195a2d988e_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b80bc4d278c647ef94c28b195a2d988e_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:28,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b09afcc3029453187a84847776a6fc7, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:28,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b09afcc3029453187a84847776a6fc7 is 175, key is test_row_0/A:col10/1731577587189/Put/seqid=0 2024-11-14T09:46:28,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742395_1571 (size=31255) 2024-11-14T09:46:28,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577648152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577648160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577648160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577648160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:46:28,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577648461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577648470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577648470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577648470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,537 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=678, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b09afcc3029453187a84847776a6fc7 2024-11-14T09:46:28,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/96c791554007484a9c09ab109226d025 is 50, key is test_row_0/B:col10/1731577587189/Put/seqid=0 2024-11-14T09:46:28,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742396_1572 (size=12301) 2024-11-14T09:46:28,959 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/96c791554007484a9c09ab109226d025 2024-11-14T09:46:28,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/84e73e1ebad8492ababd368298ba4e18 is 50, key is test_row_0/C:col10/1731577587189/Put/seqid=0 2024-11-14T09:46:28,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742397_1573 (size=12301) 2024-11-14T09:46:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47692 deadline: 1731577648980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47656 deadline: 1731577648980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47694 deadline: 1731577648980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:46:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42511 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47660 deadline: 1731577648980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:46:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., hostname=defc576eb6b7,42511,1731577408298, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:46:29,176 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-11-14T09:46:29,176 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:29,176 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,176 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:29,176 INFO [Thread-2144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2144. 2024-11-14T09:46:29,176 DEBUG [Thread-2144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:29,176 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,177 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:29,177 INFO [Thread-2140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2140. 2024-11-14T09:46:29,177 DEBUG [Thread-2140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:29,177 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,177 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:29,181 INFO [Thread-2142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2142. 2024-11-14T09:46:29,181 DEBUG [Thread-2142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:29,181 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,181 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:29,182 INFO [Thread-2138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2138. 2024-11-14T09:46:29,182 DEBUG [Thread-2138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:29,182 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,182 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:46:29,370 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/84e73e1ebad8492ababd368298ba4e18 2024-11-14T09:46:29,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/7b09afcc3029453187a84847776a6fc7 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b09afcc3029453187a84847776a6fc7 2024-11-14T09:46:29,377 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b09afcc3029453187a84847776a6fc7, entries=150, sequenceid=678, filesize=30.5 K 2024-11-14T09:46:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/96c791554007484a9c09ab109226d025 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/96c791554007484a9c09ab109226d025 2024-11-14T09:46:29,382 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/96c791554007484a9c09ab109226d025, entries=150, sequenceid=678, filesize=12.0 K 2024-11-14T09:46:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/84e73e1ebad8492ababd368298ba4e18 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/84e73e1ebad8492ababd368298ba4e18 2024-11-14T09:46:29,386 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/84e73e1ebad8492ababd368298ba4e18, entries=150, sequenceid=678, filesize=12.0 K 2024-11-14T09:46:29,393 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ee11adb61130645457ff22474228d88e in 1697ms, sequenceid=678, compaction requested=true 2024-11-14T09:46:29,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:29,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:29,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-14T09:46:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-14T09:46:29,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T09:46:29,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1970 sec 2024-11-14T09:46:29,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.2040 sec 2024-11-14T09:46:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42511 {}] regionserver.HRegion(8855): Flush requested on ee11adb61130645457ff22474228d88e 2024-11-14T09:46:29,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T09:46:29,991 INFO [Thread-2133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2133. 2024-11-14T09:46:29,991 DEBUG [Thread-2133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:29,991 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:29,991 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:29,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:29,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:29,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa3f8cd02f0f4f87b8b95c1b45e4e376_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577587832/Put/seqid=0 2024-11-14T09:46:30,002 INFO [Thread-2129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2129. 2024-11-14T09:46:30,002 DEBUG [Thread-2129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:30,002 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:30,002 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:30,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:30,002 INFO [Thread-2131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2131. 2024-11-14T09:46:30,002 DEBUG [Thread-2131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:30,002 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:30,002 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:30,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:30,003 INFO [Thread-2127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2127. 2024-11-14T09:46:30,003 DEBUG [Thread-2127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:30,003 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:30,003 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:30,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:30,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742398_1574 (size=12454) 2024-11-14T09:46:30,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:30,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa3f8cd02f0f4f87b8b95c1b45e4e376_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa3f8cd02f0f4f87b8b95c1b45e4e376_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:30,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b52842db8fb0433f8bdcef58b3f5cbef, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:30,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b52842db8fb0433f8bdcef58b3f5cbef is 175, key is test_row_0/A:col10/1731577587832/Put/seqid=0 2024-11-14T09:46:30,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742399_1575 (size=31255) 2024-11-14T09:46:30,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=690, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b52842db8fb0433f8bdcef58b3f5cbef 2024-11-14T09:46:30,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/3e4864962f414d3592bde8639eb20974 is 50, key is test_row_0/B:col10/1731577587832/Put/seqid=0 2024-11-14T09:46:30,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742400_1576 (size=12301) 2024-11-14T09:46:30,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/3e4864962f414d3592bde8639eb20974 2024-11-14T09:46:30,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/1b8039fddb21418ea25046657e37ea39 is 50, key is test_row_0/C:col10/1731577587832/Put/seqid=0 2024-11-14T09:46:30,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742401_1577 (size=12301) 2024-11-14T09:46:31,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/1b8039fddb21418ea25046657e37ea39 2024-11-14T09:46:31,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/b52842db8fb0433f8bdcef58b3f5cbef as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b52842db8fb0433f8bdcef58b3f5cbef 2024-11-14T09:46:31,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b52842db8fb0433f8bdcef58b3f5cbef, entries=150, sequenceid=690, filesize=30.5 K 2024-11-14T09:46:31,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/3e4864962f414d3592bde8639eb20974 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/3e4864962f414d3592bde8639eb20974 2024-11-14T09:46:31,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/3e4864962f414d3592bde8639eb20974, entries=150, sequenceid=690, filesize=12.0 K 2024-11-14T09:46:31,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/1b8039fddb21418ea25046657e37ea39 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/1b8039fddb21418ea25046657e37ea39 2024-11-14T09:46:31,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/1b8039fddb21418ea25046657e37ea39, entries=150, sequenceid=690, filesize=12.0 K 2024-11-14T09:46:31,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for ee11adb61130645457ff22474228d88e in 1261ms, sequenceid=690, compaction requested=true 2024-11-14T09:46:31,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:31,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:46:31,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:31,253 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:31,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:46:31,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:31,253 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:31,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee11adb61130645457ff22474228d88e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:46:31,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:31,254 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151882 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:31,254 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/A is initiating minor compaction (all files) 2024-11-14T09:46:31,255 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:31,255 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/A in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:31,255 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/B is initiating minor compaction (all files) 2024-11-14T09:46:31,255 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/B in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:31,255 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/52d95a9643f24711b5642720749d286a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dcc24aafbb104e70aed4bce128289489, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b09afcc3029453187a84847776a6fc7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b52842db8fb0433f8bdcef58b3f5cbef] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=148.3 K 2024-11-14T09:46:31,255 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f7ad23c0c80d45e09592d5661fc6b0b4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aac736d701944d53a85baac39f47170f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/96c791554007484a9c09ab109226d025, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/3e4864962f414d3592bde8639eb20974] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=48.8 K 2024-11-14T09:46:31,255 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:31,255 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. files: [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/52d95a9643f24711b5642720749d286a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dcc24aafbb104e70aed4bce128289489, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b09afcc3029453187a84847776a6fc7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b52842db8fb0433f8bdcef58b3f5cbef] 2024-11-14T09:46:31,255 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ad23c0c80d45e09592d5661fc6b0b4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731577586232 2024-11-14T09:46:31,255 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d95a9643f24711b5642720749d286a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731577586232 2024-11-14T09:46:31,255 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting aac736d701944d53a85baac39f47170f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1731577587050 2024-11-14T09:46:31,255 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcc24aafbb104e70aed4bce128289489, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1731577586887 2024-11-14T09:46:31,256 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b09afcc3029453187a84847776a6fc7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731577587181 2024-11-14T09:46:31,256 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c791554007484a9c09ab109226d025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731577587181 2024-11-14T09:46:31,256 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] compactions.Compactor(225): Compacting b52842db8fb0433f8bdcef58b3f5cbef, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=690, earliestPutTs=1731577587831 2024-11-14T09:46:31,256 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4864962f414d3592bde8639eb20974, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=690, earliestPutTs=1731577587831 2024-11-14T09:46:31,261 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:31,263 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411140fbbc6964e5149b09e1fae07b2bbe406_ee11adb61130645457ff22474228d88e store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:31,263 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:46:31,263 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ec4ffbc41cc54888ab0b16a1ee634ac4 is 50, key is test_row_0/B:col10/1731577587832/Put/seqid=0 2024-11-14T09:46:31,266 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411140fbbc6964e5149b09e1fae07b2bbe406_ee11adb61130645457ff22474228d88e, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:31,267 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140fbbc6964e5149b09e1fae07b2bbe406_ee11adb61130645457ff22474228d88e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:31,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742402_1578 (size=13221) 2024-11-14T09:46:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742403_1579 (size=4469) 2024-11-14T09:46:31,311 DEBUG [regionserver/defc576eb6b7:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-14T09:46:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T09:46:31,360 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:31,673 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ec4ffbc41cc54888ab0b16a1ee634ac4 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ec4ffbc41cc54888ab0b16a1ee634ac4 2024-11-14T09:46:31,674 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#A#compaction#481 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-14T09:46:31,675 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/ba06162f57f04cc5bb3662e9d4b63431 is 175, key is test_row_0/A:col10/1731577587832/Put/seqid=0 2024-11-14T09:46:31,677 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/B of ee11adb61130645457ff22474228d88e into ec4ffbc41cc54888ab0b16a1ee634ac4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:31,677 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:31,677 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/B, priority=12, startTime=1731577591253; duration=0sec 2024-11-14T09:46:31,677 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:46:31,677 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:B 2024-11-14T09:46:31,677 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:46:31,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742404_1580 (size=32175) 2024-11-14T09:46:31,678 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:46:31,678 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1541): ee11adb61130645457ff22474228d88e/C is initiating minor compaction (all files) 2024-11-14T09:46:31,678 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee11adb61130645457ff22474228d88e/C in TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:31,678 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/37f961a8feb1455e998ffd02a0857068, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/76cfab180a934877b6d03eb144af527b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/84e73e1ebad8492ababd368298ba4e18, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/1b8039fddb21418ea25046657e37ea39] into tmpdir=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp, totalSize=48.8 K 2024-11-14T09:46:31,678 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f961a8feb1455e998ffd02a0857068, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1731577586232 2024-11-14T09:46:31,678 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 76cfab180a934877b6d03eb144af527b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1731577587050 2024-11-14T09:46:31,679 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e73e1ebad8492ababd368298ba4e18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731577587181 2024-11-14T09:46:31,679 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8039fddb21418ea25046657e37ea39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=690, earliestPutTs=1731577587831 2024-11-14T09:46:31,686 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee11adb61130645457ff22474228d88e#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-14T09:46:31,686 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/f56e35dd6b094478a6496104e5a0b717 is 50, key is test_row_0/C:col10/1731577587832/Put/seqid=0 2024-11-14T09:46:31,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742405_1581 (size=13221) 2024-11-14T09:46:32,081 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/ba06162f57f04cc5bb3662e9d4b63431 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ba06162f57f04cc5bb3662e9d4b63431 2024-11-14T09:46:32,086 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/A of ee11adb61130645457ff22474228d88e into ba06162f57f04cc5bb3662e9d4b63431(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:32,086 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:32,086 INFO [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/A, priority=12, startTime=1731577591253; duration=0sec 2024-11-14T09:46:32,086 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:32,086 DEBUG [RS:0;defc576eb6b7:42511-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:A 2024-11-14T09:46:32,094 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/f56e35dd6b094478a6496104e5a0b717 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/f56e35dd6b094478a6496104e5a0b717 2024-11-14T09:46:32,098 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee11adb61130645457ff22474228d88e/C of ee11adb61130645457ff22474228d88e into f56e35dd6b094478a6496104e5a0b717(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:46:32,098 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee11adb61130645457ff22474228d88e: 2024-11-14T09:46:32,098 INFO [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., storeName=ee11adb61130645457ff22474228d88e/C, priority=12, startTime=1731577591253; duration=0sec 2024-11-14T09:46:32,098 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:46:32,098 DEBUG [RS:0;defc576eb6b7:42511-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee11adb61130645457ff22474228d88e:C 2024-11-14T09:46:37,651 INFO [Thread-2125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2125. 2024-11-14T09:46:37,651 DEBUG [Thread-2125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:46:37,651 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:37,651 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:37,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 178 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2536 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2711 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2772 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2544 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2709 2024-11-14T09:46:37,652 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:46:37,652 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:46:37,652 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:46:37,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:37,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:37,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:37,653 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:46:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:46:37,655 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577597655"}]},"ts":"1731577597655"} 2024-11-14T09:46:37,656 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:46:37,656 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:46:37,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:46:37,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, UNASSIGN}] 2024-11-14T09:46:37,659 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, UNASSIGN 2024-11-14T09:46:37,659 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=150 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=CLOSING, regionLocation=defc576eb6b7,42511,1731577408298 2024-11-14T09:46:37,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, UNASSIGN because future has completed 2024-11-14T09:46:37,660 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:46:37,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298}] 2024-11-14T09:46:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:46:37,812 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(122): Close ee11adb61130645457ff22474228d88e 2024-11-14T09:46:37,812 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:46:37,812 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1722): Closing ee11adb61130645457ff22474228d88e, disabling compactions & flushes 2024-11-14T09:46:37,812 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:37,812 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:37,812 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. after waiting 0 ms 2024-11-14T09:46:37,812 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:37,812 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(2902): Flushing ee11adb61130645457ff22474228d88e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T09:46:37,813 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=A 2024-11-14T09:46:37,813 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:37,813 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=B 2024-11-14T09:46:37,813 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:37,813 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee11adb61130645457ff22474228d88e, store=C 2024-11-14T09:46:37,813 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T09:46:37,817 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140b0145177d02415f83d5dc3ee5fa5837_ee11adb61130645457ff22474228d88e is 50, key is test_row_0/A:col10/1731577597650/Put/seqid=0 2024-11-14T09:46:37,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742406_1582 (size=9914) 2024-11-14T09:46:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:46:38,220 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:46:38,223 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140b0145177d02415f83d5dc3ee5fa5837_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140b0145177d02415f83d5dc3ee5fa5837_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:38,224 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/769c9ac6095a44419aac6c099df2538a, store: [table=TestAcidGuarantees family=A region=ee11adb61130645457ff22474228d88e] 2024-11-14T09:46:38,224 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/769c9ac6095a44419aac6c099df2538a is 175, key is test_row_0/A:col10/1731577597650/Put/seqid=0 2024-11-14T09:46:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742407_1583 (size=22561) 2024-11-14T09:46:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:46:38,630 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=700, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/769c9ac6095a44419aac6c099df2538a 2024-11-14T09:46:38,636 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ceb1360bb69f49d09c68e6269f18284f is 50, key is test_row_0/B:col10/1731577597650/Put/seqid=0 2024-11-14T09:46:38,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742408_1584 (size=9857) 2024-11-14T09:46:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:46:39,040 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=700 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ceb1360bb69f49d09c68e6269f18284f 2024-11-14T09:46:39,051 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/d57115d40fd34f5a9d9fde6bd756245c is 50, key is test_row_0/C:col10/1731577597650/Put/seqid=0 2024-11-14T09:46:39,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742409_1585 (size=9857) 2024-11-14T09:46:39,455 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=700 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/d57115d40fd34f5a9d9fde6bd756245c 2024-11-14T09:46:39,458 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/A/769c9ac6095a44419aac6c099df2538a as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/769c9ac6095a44419aac6c099df2538a 2024-11-14T09:46:39,461 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/769c9ac6095a44419aac6c099df2538a, entries=100, sequenceid=700, filesize=22.0 K 2024-11-14T09:46:39,462 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/B/ceb1360bb69f49d09c68e6269f18284f as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb1360bb69f49d09c68e6269f18284f 2024-11-14T09:46:39,465 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb1360bb69f49d09c68e6269f18284f, entries=100, sequenceid=700, filesize=9.6 K 2024-11-14T09:46:39,465 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/.tmp/C/d57115d40fd34f5a9d9fde6bd756245c as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d57115d40fd34f5a9d9fde6bd756245c 2024-11-14T09:46:39,468 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d57115d40fd34f5a9d9fde6bd756245c, entries=100, sequenceid=700, filesize=9.6 K 2024-11-14T09:46:39,469 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ee11adb61130645457ff22474228d88e in 1656ms, sequenceid=700, compaction requested=false 2024-11-14T09:46:39,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b4862536059422e964bf60f7d748989, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ac30964097f44a38b55574b43dd9cd08, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/eceb93dc61a743088f758c26e45c1945, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6723d7b0059e4dd69c96917aebeed90b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/06b112bcf7f947679c71928765a774f5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cd627391288b4450b7da8e7e8d639b57, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5f9d02278ada488b9b55d9ef017b1972, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6e9f085eebcd42f1a917cb4138f1ea08, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cdf5a564ee4e43e29ac008911130feb8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/07070dc7d8d74590beaff25a77a6bb1a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/69d7c10e09f24cb5a36cd217ca608b0d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5986691ce4a34d408d0ad232e08cf542, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/8c06464837b84a8b9e3247cf80627d6f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/3d172378d56c4755bca623ddb626af21, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c0ab732295e440258eef4ebdee345f8c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/82303ae2c2df44d9ab6410dbe6c67697, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c89c46bdd26c4d4393bcb9e35ef5d57c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/26d6216a95684fb49fcbdd6431bec0ce, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b06842588cf743ef99e1f8fb90f0fa4c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cfcfda0465bc4146a503188d9ce68c58, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5b3153e93e4a48e9aff14f59e4d6affb, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dd285bf639b5482aace3d6ae8d512101, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/52d95a9643f24711b5642720749d286a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6ed9aefed7c74c77abcb562641d8e72f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dcc24aafbb104e70aed4bce128289489, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b09afcc3029453187a84847776a6fc7, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b52842db8fb0433f8bdcef58b3f5cbef] to archive 2024-11-14T09:46:39,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:46:39,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b4862536059422e964bf60f7d748989 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b4862536059422e964bf60f7d748989 2024-11-14T09:46:39,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ac30964097f44a38b55574b43dd9cd08 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ac30964097f44a38b55574b43dd9cd08 2024-11-14T09:46:39,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/eceb93dc61a743088f758c26e45c1945 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/eceb93dc61a743088f758c26e45c1945 2024-11-14T09:46:39,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6723d7b0059e4dd69c96917aebeed90b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6723d7b0059e4dd69c96917aebeed90b 2024-11-14T09:46:39,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/06b112bcf7f947679c71928765a774f5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/06b112bcf7f947679c71928765a774f5 2024-11-14T09:46:39,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cd627391288b4450b7da8e7e8d639b57 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cd627391288b4450b7da8e7e8d639b57 2024-11-14T09:46:39,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5f9d02278ada488b9b55d9ef017b1972 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5f9d02278ada488b9b55d9ef017b1972 2024-11-14T09:46:39,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6e9f085eebcd42f1a917cb4138f1ea08 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6e9f085eebcd42f1a917cb4138f1ea08 2024-11-14T09:46:39,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cdf5a564ee4e43e29ac008911130feb8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cdf5a564ee4e43e29ac008911130feb8 2024-11-14T09:46:39,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/07070dc7d8d74590beaff25a77a6bb1a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/07070dc7d8d74590beaff25a77a6bb1a 2024-11-14T09:46:39,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/69d7c10e09f24cb5a36cd217ca608b0d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/69d7c10e09f24cb5a36cd217ca608b0d 2024-11-14T09:46:39,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5986691ce4a34d408d0ad232e08cf542 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5986691ce4a34d408d0ad232e08cf542 2024-11-14T09:46:39,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/8c06464837b84a8b9e3247cf80627d6f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/8c06464837b84a8b9e3247cf80627d6f 2024-11-14T09:46:39,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/3d172378d56c4755bca623ddb626af21 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/3d172378d56c4755bca623ddb626af21 2024-11-14T09:46:39,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c0ab732295e440258eef4ebdee345f8c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c0ab732295e440258eef4ebdee345f8c 2024-11-14T09:46:39,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/82303ae2c2df44d9ab6410dbe6c67697 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/82303ae2c2df44d9ab6410dbe6c67697 2024-11-14T09:46:39,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c89c46bdd26c4d4393bcb9e35ef5d57c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/c89c46bdd26c4d4393bcb9e35ef5d57c 2024-11-14T09:46:39,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/26d6216a95684fb49fcbdd6431bec0ce to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/26d6216a95684fb49fcbdd6431bec0ce 2024-11-14T09:46:39,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b06842588cf743ef99e1f8fb90f0fa4c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b06842588cf743ef99e1f8fb90f0fa4c 2024-11-14T09:46:39,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cfcfda0465bc4146a503188d9ce68c58 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/cfcfda0465bc4146a503188d9ce68c58 2024-11-14T09:46:39,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5b3153e93e4a48e9aff14f59e4d6affb to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/5b3153e93e4a48e9aff14f59e4d6affb 2024-11-14T09:46:39,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dd285bf639b5482aace3d6ae8d512101 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dd285bf639b5482aace3d6ae8d512101 2024-11-14T09:46:39,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/52d95a9643f24711b5642720749d286a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/52d95a9643f24711b5642720749d286a 2024-11-14T09:46:39,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6ed9aefed7c74c77abcb562641d8e72f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/6ed9aefed7c74c77abcb562641d8e72f 2024-11-14T09:46:39,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dcc24aafbb104e70aed4bce128289489 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/dcc24aafbb104e70aed4bce128289489 2024-11-14T09:46:39,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b09afcc3029453187a84847776a6fc7 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/7b09afcc3029453187a84847776a6fc7 2024-11-14T09:46:39,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b52842db8fb0433f8bdcef58b3f5cbef to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/b52842db8fb0433f8bdcef58b3f5cbef 2024-11-14T09:46:39,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e6fbf14940cd44db8e2af9df9dfcc0b8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/26d140d6e2b54ea09c32e13bc785606b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/9b2058cddb60499d96b97709aacd6169, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/2565e9bc02214640b4b867cb3112b30a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/73aa7c509967459bb174a67b3b860fc6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f493eb50e33144159960288adac56969, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e57630b4d84b4dbd037d10c469a937, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/692dc8cb64164b16b13c5e8a24cedd03, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e70869c3aa48c3b4f7e70475e1debc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/d42d19c664454d2590314a539fb5a70d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/33006ce292604b4f878e19e8d8742d0a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/0e8c024735d94fd9844c67c9cb3ddbf9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aa7b88cedde842fabffcddb92381a240, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb4e5d2276a44f091feff2dd0b06fe3, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/a098ba265e7f4eb9986d2cdc5cf6fa74, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aec83e3f2ff243b3ab1dc8f71eb42ba8, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ccea419231be4a868329404a6e4e5b6e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/48e9967f37844a22a6818a9bf0c813ca, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f8b18740b58e481a9486cdc421ecce04, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/6150f30d57534d84937a732964afac31, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/08dd143226d945738e6a7b7ecce7d57c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/b5f1773eadcd455b883f1a30239d6364, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f7ad23c0c80d45e09592d5661fc6b0b4, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/cbfec3ba3db74edbbb5a3082b2d452c6, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aac736d701944d53a85baac39f47170f, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/96c791554007484a9c09ab109226d025, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/3e4864962f414d3592bde8639eb20974] to archive 2024-11-14T09:46:39,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:46:39,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e6fbf14940cd44db8e2af9df9dfcc0b8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e6fbf14940cd44db8e2af9df9dfcc0b8 2024-11-14T09:46:39,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/26d140d6e2b54ea09c32e13bc785606b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/26d140d6e2b54ea09c32e13bc785606b 2024-11-14T09:46:39,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/9b2058cddb60499d96b97709aacd6169 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/9b2058cddb60499d96b97709aacd6169 2024-11-14T09:46:39,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/2565e9bc02214640b4b867cb3112b30a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/2565e9bc02214640b4b867cb3112b30a 2024-11-14T09:46:39,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/73aa7c509967459bb174a67b3b860fc6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/73aa7c509967459bb174a67b3b860fc6 2024-11-14T09:46:39,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f493eb50e33144159960288adac56969 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f493eb50e33144159960288adac56969 2024-11-14T09:46:39,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e57630b4d84b4dbd037d10c469a937 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e57630b4d84b4dbd037d10c469a937 2024-11-14T09:46:39,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/692dc8cb64164b16b13c5e8a24cedd03 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/692dc8cb64164b16b13c5e8a24cedd03 2024-11-14T09:46:39,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e70869c3aa48c3b4f7e70475e1debc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/e2e70869c3aa48c3b4f7e70475e1debc 2024-11-14T09:46:39,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/d42d19c664454d2590314a539fb5a70d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/d42d19c664454d2590314a539fb5a70d 2024-11-14T09:46:39,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/33006ce292604b4f878e19e8d8742d0a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/33006ce292604b4f878e19e8d8742d0a 2024-11-14T09:46:39,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/0e8c024735d94fd9844c67c9cb3ddbf9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/0e8c024735d94fd9844c67c9cb3ddbf9 2024-11-14T09:46:39,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aa7b88cedde842fabffcddb92381a240 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aa7b88cedde842fabffcddb92381a240 2024-11-14T09:46:39,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb4e5d2276a44f091feff2dd0b06fe3 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb4e5d2276a44f091feff2dd0b06fe3 2024-11-14T09:46:39,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/a098ba265e7f4eb9986d2cdc5cf6fa74 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/a098ba265e7f4eb9986d2cdc5cf6fa74 2024-11-14T09:46:39,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aec83e3f2ff243b3ab1dc8f71eb42ba8 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aec83e3f2ff243b3ab1dc8f71eb42ba8 2024-11-14T09:46:39,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ccea419231be4a868329404a6e4e5b6e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ccea419231be4a868329404a6e4e5b6e 2024-11-14T09:46:39,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/48e9967f37844a22a6818a9bf0c813ca to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/48e9967f37844a22a6818a9bf0c813ca 2024-11-14T09:46:39,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f8b18740b58e481a9486cdc421ecce04 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f8b18740b58e481a9486cdc421ecce04 2024-11-14T09:46:39,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/6150f30d57534d84937a732964afac31 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/6150f30d57534d84937a732964afac31 2024-11-14T09:46:39,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/08dd143226d945738e6a7b7ecce7d57c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/08dd143226d945738e6a7b7ecce7d57c 2024-11-14T09:46:39,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/b5f1773eadcd455b883f1a30239d6364 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/b5f1773eadcd455b883f1a30239d6364 2024-11-14T09:46:39,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f7ad23c0c80d45e09592d5661fc6b0b4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/f7ad23c0c80d45e09592d5661fc6b0b4 2024-11-14T09:46:39,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/cbfec3ba3db74edbbb5a3082b2d452c6 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/cbfec3ba3db74edbbb5a3082b2d452c6 2024-11-14T09:46:39,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aac736d701944d53a85baac39f47170f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/aac736d701944d53a85baac39f47170f 2024-11-14T09:46:39,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/96c791554007484a9c09ab109226d025 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/96c791554007484a9c09ab109226d025 2024-11-14T09:46:39,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/3e4864962f414d3592bde8639eb20974 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/3e4864962f414d3592bde8639eb20974 2024-11-14T09:46:39,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8adb14780b784c8099149242e06a8180, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/49632e8cae5841fe923cf86e28929763, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d474996986f34e1ebc97f80f78b801c5, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c9a8fc2fc83b4d6bb9c0e89329f01247, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7a84ab006670456e851a299a6328d248, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9bc053e4e13447308a0150badd2cf6c2, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/abca163916c343d7a2c2596d4aa0ac6d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ef09a6fadfa241bf94fe46f89e65d25c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/fafccfde177349609acf84530fa0d0ba, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7562052032f04e3986a8aaada84792a9, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8cc5e28dda46452c93f0d84d26dbb17d, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/4f5dc883a1654bf788fa28e9771e082c, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9f7a0330c6b64ebd8a761b6d84fcab42, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ba04c75d547c4568add173b744da6fad, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c24fc16b1d30463f95c975c54ca8f2db, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c567080dd0214584b30b5c2ae3f7a5df, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d03748bebe464c98bcba43a0cad600ac, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/929a4883b61645fabb6370e6e71a07dc, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/96375d46d9314966b75bcb431f4b802b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/a913f2bab53e4526ae9c6881d0f66c1e, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/b6b2294a010543b29064b6c8a274e12b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8c7169d5042e48f6ae4f9f4c86aaab38, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/37f961a8feb1455e998ffd02a0857068, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/bee97cc1445245afb684e88f2964b03a, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/76cfab180a934877b6d03eb144af527b, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/84e73e1ebad8492ababd368298ba4e18, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/1b8039fddb21418ea25046657e37ea39] to archive 2024-11-14T09:46:39,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:46:39,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8adb14780b784c8099149242e06a8180 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8adb14780b784c8099149242e06a8180 2024-11-14T09:46:39,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/49632e8cae5841fe923cf86e28929763 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/49632e8cae5841fe923cf86e28929763 2024-11-14T09:46:39,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d474996986f34e1ebc97f80f78b801c5 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d474996986f34e1ebc97f80f78b801c5 2024-11-14T09:46:39,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c9a8fc2fc83b4d6bb9c0e89329f01247 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c9a8fc2fc83b4d6bb9c0e89329f01247 2024-11-14T09:46:39,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7a84ab006670456e851a299a6328d248 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7a84ab006670456e851a299a6328d248 2024-11-14T09:46:39,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9bc053e4e13447308a0150badd2cf6c2 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9bc053e4e13447308a0150badd2cf6c2 2024-11-14T09:46:39,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/abca163916c343d7a2c2596d4aa0ac6d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/abca163916c343d7a2c2596d4aa0ac6d 2024-11-14T09:46:39,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ef09a6fadfa241bf94fe46f89e65d25c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ef09a6fadfa241bf94fe46f89e65d25c 2024-11-14T09:46:39,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/fafccfde177349609acf84530fa0d0ba to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/fafccfde177349609acf84530fa0d0ba 2024-11-14T09:46:39,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7562052032f04e3986a8aaada84792a9 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/7562052032f04e3986a8aaada84792a9 2024-11-14T09:46:39,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8cc5e28dda46452c93f0d84d26dbb17d to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8cc5e28dda46452c93f0d84d26dbb17d 2024-11-14T09:46:39,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/4f5dc883a1654bf788fa28e9771e082c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/4f5dc883a1654bf788fa28e9771e082c 2024-11-14T09:46:39,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9f7a0330c6b64ebd8a761b6d84fcab42 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/9f7a0330c6b64ebd8a761b6d84fcab42 2024-11-14T09:46:39,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ba04c75d547c4568add173b744da6fad to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/ba04c75d547c4568add173b744da6fad 2024-11-14T09:46:39,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c24fc16b1d30463f95c975c54ca8f2db to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c24fc16b1d30463f95c975c54ca8f2db 2024-11-14T09:46:39,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c567080dd0214584b30b5c2ae3f7a5df to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/c567080dd0214584b30b5c2ae3f7a5df 2024-11-14T09:46:39,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d03748bebe464c98bcba43a0cad600ac to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d03748bebe464c98bcba43a0cad600ac 2024-11-14T09:46:39,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/929a4883b61645fabb6370e6e71a07dc to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/929a4883b61645fabb6370e6e71a07dc 2024-11-14T09:46:39,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/96375d46d9314966b75bcb431f4b802b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/96375d46d9314966b75bcb431f4b802b 2024-11-14T09:46:39,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/a913f2bab53e4526ae9c6881d0f66c1e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/a913f2bab53e4526ae9c6881d0f66c1e 2024-11-14T09:46:39,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/b6b2294a010543b29064b6c8a274e12b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/b6b2294a010543b29064b6c8a274e12b 2024-11-14T09:46:39,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8c7169d5042e48f6ae4f9f4c86aaab38 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/8c7169d5042e48f6ae4f9f4c86aaab38 2024-11-14T09:46:39,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/37f961a8feb1455e998ffd02a0857068 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/37f961a8feb1455e998ffd02a0857068 2024-11-14T09:46:39,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/bee97cc1445245afb684e88f2964b03a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/bee97cc1445245afb684e88f2964b03a 2024-11-14T09:46:39,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/76cfab180a934877b6d03eb144af527b to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/76cfab180a934877b6d03eb144af527b 2024-11-14T09:46:39,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/84e73e1ebad8492ababd368298ba4e18 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/84e73e1ebad8492ababd368298ba4e18 2024-11-14T09:46:39,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/1b8039fddb21418ea25046657e37ea39 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/1b8039fddb21418ea25046657e37ea39 2024-11-14T09:46:39,559 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/recovered.edits/703.seqid, newMaxSeqId=703, maxSeqId=4 2024-11-14T09:46:39,559 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. 2024-11-14T09:46:39,559 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1676): Region close journal for ee11adb61130645457ff22474228d88e: Waiting for close lock at 1731577597812Running coprocessor pre-close hooks at 1731577597812Disabling compacts and flushes for region at 1731577597812Disabling writes for close at 1731577597812Obtaining lock to block concurrent updates at 1731577597812Preparing flush snapshotting stores in ee11adb61130645457ff22474228d88e at 1731577597812Finished memstore snapshotting TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731577597813 (+1 ms)Flushing stores of TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e. at 1731577597813Flushing ee11adb61130645457ff22474228d88e/A: creating writer at 1731577597813Flushing mob file ee11adb61130645457ff22474228d88e/A: appending metadata at 1731577597817 (+4 ms)Flushing mob file ee11adb61130645457ff22474228d88e/A: closing flushed file at 1731577597817Flushing ee11adb61130645457ff22474228d88e/A: appending metadata at 1731577598224 (+407 ms)Flushing ee11adb61130645457ff22474228d88e/A: closing flushed file at 1731577598224Flushing ee11adb61130645457ff22474228d88e/B: creating writer at 1731577598633 (+409 ms)Flushing ee11adb61130645457ff22474228d88e/B: appending metadata at 1731577598635 (+2 ms)Flushing ee11adb61130645457ff22474228d88e/B: closing flushed file at 1731577598635Flushing ee11adb61130645457ff22474228d88e/C: creating writer at 1731577599047 (+412 ms)Flushing ee11adb61130645457ff22474228d88e/C: appending metadata at 1731577599050 (+3 ms)Flushing ee11adb61130645457ff22474228d88e/C: closing flushed file at 1731577599050Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d1eea0c: reopening flushed file at 1731577599458 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3230c9fd: reopening flushed file at 1731577599461 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@699addd3: reopening flushed file at 1731577599465 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ee11adb61130645457ff22474228d88e in 1656ms, sequenceid=700, compaction requested=false at 1731577599469 (+4 ms)Writing region close event to WAL at 1731577599556 (+87 ms)Running coprocessor post-close hooks at 1731577599559 (+3 ms)Closed at 1731577599559 2024-11-14T09:46:39,561 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(157): Closed ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,561 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=150 updating hbase:meta row=ee11adb61130645457ff22474228d88e, regionState=CLOSED 2024-11-14T09:46:39,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 because future has completed 2024-11-14T09:46:39,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T09:46:39,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; CloseRegionProcedure ee11adb61130645457ff22474228d88e, server=defc576eb6b7,42511,1731577408298 in 1.9030 sec 2024-11-14T09:46:39,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-14T09:46:39,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee11adb61130645457ff22474228d88e, UNASSIGN in 1.9060 sec 2024-11-14T09:46:39,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T09:46:39,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9090 sec 2024-11-14T09:46:39,568 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577599567"}]},"ts":"1731577599567"} 2024-11-14T09:46:39,569 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:46:39,569 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:46:39,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-14T09:46:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T09:46:39,801 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:39,801 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:46:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:39,803 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:46:39,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=152, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:39,807 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,809 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C, FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/recovered.edits] 2024-11-14T09:46:39,811 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/769c9ac6095a44419aac6c099df2538a to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/769c9ac6095a44419aac6c099df2538a 2024-11-14T09:46:39,812 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ba06162f57f04cc5bb3662e9d4b63431 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/A/ba06162f57f04cc5bb3662e9d4b63431 2024-11-14T09:46:39,814 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb1360bb69f49d09c68e6269f18284f to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ceb1360bb69f49d09c68e6269f18284f 2024-11-14T09:46:39,816 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ec4ffbc41cc54888ab0b16a1ee634ac4 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/B/ec4ffbc41cc54888ab0b16a1ee634ac4 2024-11-14T09:46:39,818 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d57115d40fd34f5a9d9fde6bd756245c to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/d57115d40fd34f5a9d9fde6bd756245c 2024-11-14T09:46:39,819 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/f56e35dd6b094478a6496104e5a0b717 to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/C/f56e35dd6b094478a6496104e5a0b717 2024-11-14T09:46:39,822 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/recovered.edits/703.seqid to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e/recovered.edits/703.seqid 2024-11-14T09:46:39,823 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/default/TestAcidGuarantees/ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,823 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:46:39,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:46:39,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:46:39,828 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140b0145177d02415f83d5dc3ee5fa5837_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140b0145177d02415f83d5dc3ee5fa5837_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,829 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140c2b2f5d7926408fb0e80fdea1e3c9cc_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140c2b2f5d7926408fb0e80fdea1e3c9cc_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,830 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140faba77dfbcf44a68bf4ad2549ee9adb_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140faba77dfbcf44a68bf4ad2549ee9adb_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,832 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141eb13d8a38314dbf988533a9048e92c1_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141eb13d8a38314dbf988533a9048e92c1_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,834 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111441878a501a3f4f7e9e9afb9a9f7be51f_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111441878a501a3f4f7e9e9afb9a9f7be51f_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,835 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145aacd138c1fc4e0cafa2cbb8319c64cf_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145aacd138c1fc4e0cafa2cbb8319c64cf_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,836 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145e97588a96994174baca4766eb7d20a3_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145e97588a96994174baca4766eb7d20a3_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,837 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114602765018c6b4ea197f482ce0d48f8a7_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114602765018c6b4ea197f482ce0d48f8a7_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,838 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146bf460ca19644708ada7fae7af23e879_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146bf460ca19644708ada7fae7af23e879_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,839 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148c51386c78dd48978026d5f4662f7432_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148c51386c78dd48978026d5f4662f7432_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,839 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114990b31e0eb904c0d8eecc0aeb3ce900e_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114990b31e0eb904c0d8eecc0aeb3ce900e_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,840 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1212978e4df4d9fa1fca02ab8e08cc0_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b1212978e4df4d9fa1fca02ab8e08cc0_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,842 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b533c3a3bb5d4b5097b295dc256b86bb_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b533c3a3bb5d4b5097b295dc256b86bb_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,843 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b6f240d374014d7cab991d40321a6bd8_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b6f240d374014d7cab991d40321a6bd8_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,845 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b80bc4d278c647ef94c28b195a2d988e_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b80bc4d278c647ef94c28b195a2d988e_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,846 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ccdebcf5afdc42ee81e6f8885c65da22_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ccdebcf5afdc42ee81e6f8885c65da22_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,847 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d67b50b9ffcd43b88b513e95adf4d14c_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d67b50b9ffcd43b88b513e95adf4d14c_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,847 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dc00bc244b064911993453101bd434bf_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dc00bc244b064911993453101bd434bf_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,848 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e482db9157cc48e9b271322c619ee2f1_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e482db9157cc48e9b271322c619ee2f1_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,849 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb9aa90c103840c9a24d175772eae5f5_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb9aa90c103840c9a24d175772eae5f5_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,850 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa3f8cd02f0f4f87b8b95c1b45e4e376_ee11adb61130645457ff22474228d88e to hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa3f8cd02f0f4f87b8b95c1b45e4e376_ee11adb61130645457ff22474228d88e 2024-11-14T09:46:39,850 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:46:39,852 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=152, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:39,854 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:46:39,858 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:46:39,859 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=152, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:39,859 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:46:39,859 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577599859"}]},"ts":"9223372036854775807"} 2024-11-14T09:46:39,861 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:46:39,862 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee11adb61130645457ff22474228d88e, NAME => 'TestAcidGuarantees,,1731577565901.ee11adb61130645457ff22474228d88e.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:46:39,862 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:46:39,862 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577599862"}]},"ts":"9223372036854775807"} 2024-11-14T09:46:39,864 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:46:39,865 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=152, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:46:39,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-14T09:46:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:46:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:46:39,911 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:46:39,921 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=467 (was 328) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3493 (was 3651) 2024-11-14T09:46:39,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T09:46:39,921 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:46:39,921 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:46:39,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:39,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:39,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:46:39,922 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T09:46:39,922 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-11-14T09:46:39,922 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=defc576eb6b7,33347,1731577407591 2024-11-14T09:46:39,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:46:39,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:46:39,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:46:39,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:46:39,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:46:39,963 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:46:39,963 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:46:39,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:39,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:46:39,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:46:39,963 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'defc576eb6b7,42511,1731577408298' ***** 2024-11-14T09:46:39,963 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T09:46:39,964 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T09:46:39,964 INFO [RS:0;defc576eb6b7:42511 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T09:46:39,964 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T09:46:39,964 INFO [RS:0;defc576eb6b7:42511 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T09:46:39,964 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(959): stopping server defc576eb6b7,42511,1731577408298 2024-11-14T09:46:39,964 INFO [RS:0;defc576eb6b7:42511 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:46:39,964 INFO [RS:0;defc576eb6b7:42511 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;defc576eb6b7:42511. 2024-11-14T09:46:39,964 DEBUG [RS:0;defc576eb6b7:42511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:46:39,964 DEBUG [RS:0;defc576eb6b7:42511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:39,965 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T09:46:39,965 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T09:46:39,965 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T09:46:39,965 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T09:46:39,965 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T09:46:39,965 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T09:46:39,965 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:39,965 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:46:39,966 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:46:39,966 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:46:39,966 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:46:39,966 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:46:39,966 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T09:46:39,972 INFO [regionserver/defc576eb6b7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:46:39,986 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/info/43bb7529f18f40c49f5bf172db591073 is 95, key is TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b./info:/1731577439858/DeleteFamily/seqid=0 2024-11-14T09:46:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742410_1586 (size=6514) 2024-11-14T09:46:40,166 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:40,274 INFO [regionserver/defc576eb6b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T09:46:40,275 INFO [regionserver/defc576eb6b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T09:46:40,366 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:40,390 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/info/43bb7529f18f40c49f5bf172db591073 2024-11-14T09:46:40,410 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/ns/36c205d86d014097bb00602243abf196 is 93, key is TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b./ns:/1731577439810/DeleteFamily/seqid=0 2024-11-14T09:46:40,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742411_1587 (size=5906) 2024-11-14T09:46:40,566 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:40,766 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:40,813 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/ns/36c205d86d014097bb00602243abf196 2024-11-14T09:46:40,832 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/rep_barrier/79e923107a5d4d2c9d57fc25e5ef67f6 is 102, key is TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b./rep_barrier:/1731577439810/DeleteFamily/seqid=0 2024-11-14T09:46:40,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742412_1588 (size=6025) 2024-11-14T09:46:40,967 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:41,167 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:41,236 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/rep_barrier/79e923107a5d4d2c9d57fc25e5ef67f6 2024-11-14T09:46:41,261 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/table/5be7a8d593b442cca3763b73f42d0b23 is 96, key is TestAcidGuarantees,,1731577412906.52ef1722f0e126ebd82cd709b2dcdf9b./table:/1731577439810/DeleteFamily/seqid=0 2024-11-14T09:46:41,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742413_1589 (size=5936) 2024-11-14T09:46:41,367 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:41,567 DEBUG [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:46:41,665 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/table/5be7a8d593b442cca3763b73f42d0b23 2024-11-14T09:46:41,674 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/info/43bb7529f18f40c49f5bf172db591073 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/info/43bb7529f18f40c49f5bf172db591073 2024-11-14T09:46:41,680 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/info/43bb7529f18f40c49f5bf172db591073, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T09:46:41,682 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/ns/36c205d86d014097bb00602243abf196 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/ns/36c205d86d014097bb00602243abf196 2024-11-14T09:46:41,685 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/ns/36c205d86d014097bb00602243abf196, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T09:46:41,686 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/rep_barrier/79e923107a5d4d2c9d57fc25e5ef67f6 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/rep_barrier/79e923107a5d4d2c9d57fc25e5ef67f6 2024-11-14T09:46:41,689 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/rep_barrier/79e923107a5d4d2c9d57fc25e5ef67f6, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T09:46:41,690 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/.tmp/table/5be7a8d593b442cca3763b73f42d0b23 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/table/5be7a8d593b442cca3763b73f42d0b23 2024-11-14T09:46:41,693 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/table/5be7a8d593b442cca3763b73f42d0b23, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T09:46:41,694 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1728ms, sequenceid=90, compaction requested=false 2024-11-14T09:46:41,699 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T09:46:41,700 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:46:41,700 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:46:41,700 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731577599965Running coprocessor pre-close hooks at 1731577599965Disabling compacts and flushes for region at 1731577599965Disabling writes for close at 1731577599966 (+1 ms)Obtaining lock to block concurrent updates at 1731577599966Preparing flush snapshotting stores in 1588230740 at 1731577599966Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731577599966Flushing stores of hbase:meta,,1.1588230740 at 1731577599967 (+1 ms)Flushing 1588230740/info: creating writer at 1731577599967Flushing 1588230740/info: appending metadata at 1731577599986 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731577599986Flushing 1588230740/ns: creating writer at 1731577600394 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731577600409 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731577600409Flushing 1588230740/rep_barrier: creating writer at 1731577600817 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731577600831 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731577600831Flushing 1588230740/table: creating writer at 1731577601241 (+410 ms)Flushing 1588230740/table: appending metadata at 1731577601261 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731577601261Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f294768: reopening flushed file at 1731577601672 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9a36262: reopening flushed file at 1731577601681 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@745fb9b: reopening flushed file at 1731577601685 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@780e6b9a: reopening flushed file at 1731577601689 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1728ms, sequenceid=90, compaction requested=false at 1731577601694 (+5 ms)Writing region close event to WAL at 1731577601696 (+2 ms)Running coprocessor post-close hooks at 1731577601700 (+4 ms)Closed at 1731577601700 2024-11-14T09:46:41,701 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T09:46:41,768 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(976): stopping server defc576eb6b7,42511,1731577408298; all regions closed. 2024-11-14T09:46:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741834_1010 (size=25344) 2024-11-14T09:46:41,779 DEBUG [RS:0;defc576eb6b7:42511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/oldWALs 2024-11-14T09:46:41,779 INFO [RS:0;defc576eb6b7:42511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL defc576eb6b7%2C42511%2C1731577408298.meta:.meta(num 1731577412490) 2024-11-14T09:46:41,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741832_1008 (size=18079635) 2024-11-14T09:46:41,785 DEBUG [RS:0;defc576eb6b7:42511 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/oldWALs 2024-11-14T09:46:41,785 INFO [RS:0;defc576eb6b7:42511 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL defc576eb6b7%2C42511%2C1731577408298:(num 1731577411418) 2024-11-14T09:46:41,785 DEBUG [RS:0;defc576eb6b7:42511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:46:41,785 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:46:41,786 INFO [RS:0;defc576eb6b7:42511 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:46:41,786 INFO [RS:0;defc576eb6b7:42511 {}] hbase.ChoreService(370): Chore service for: regionserver/defc576eb6b7:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T09:46:41,786 INFO [RS:0;defc576eb6b7:42511 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:46:41,786 INFO [regionserver/defc576eb6b7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:46:41,788 INFO [RS:0;defc576eb6b7:42511 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42511 2024-11-14T09:46:41,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:46:41,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/defc576eb6b7,42511,1731577408298 2024-11-14T09:46:41,846 INFO [RS:0;defc576eb6b7:42511 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:46:41,857 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [defc576eb6b7,42511,1731577408298] 2024-11-14T09:46:41,867 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/defc576eb6b7,42511,1731577408298 already deleted, retry=false 2024-11-14T09:46:41,867 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; defc576eb6b7,42511,1731577408298 expired; onlineServers=0 2024-11-14T09:46:41,867 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'defc576eb6b7,33347,1731577407591' ***** 2024-11-14T09:46:41,867 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T09:46:41,867 INFO [M:0;defc576eb6b7:33347 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:46:41,867 INFO [M:0;defc576eb6b7:33347 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:46:41,867 DEBUG [M:0;defc576eb6b7:33347 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T09:46:41,868 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T09:46:41,868 DEBUG [M:0;defc576eb6b7:33347 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T09:46:41,868 DEBUG [master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.small.0-1731577411162 {}] cleaner.HFileCleaner(306): Exit Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.small.0-1731577411162,5,FailOnTimeoutGroup] 2024-11-14T09:46:41,868 DEBUG [master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.large.0-1731577411160 {}] cleaner.HFileCleaner(306): Exit Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.large.0-1731577411160,5,FailOnTimeoutGroup] 2024-11-14T09:46:41,868 INFO [M:0;defc576eb6b7:33347 {}] hbase.ChoreService(370): Chore service for: master/defc576eb6b7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T09:46:41,868 INFO [M:0;defc576eb6b7:33347 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:46:41,868 DEBUG [M:0;defc576eb6b7:33347 {}] master.HMaster(1795): Stopping service threads 2024-11-14T09:46:41,868 INFO [M:0;defc576eb6b7:33347 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T09:46:41,868 INFO [M:0;defc576eb6b7:33347 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:46:41,868 ERROR [M:0;defc576eb6b7:33347 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:35709 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35709,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T09:46:41,869 INFO [M:0;defc576eb6b7:33347 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T09:46:41,869 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T09:46:41,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T09:46:41,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:46:41,878 DEBUG [M:0;defc576eb6b7:33347 {}] zookeeper.ZKUtil(347): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T09:46:41,878 WARN [M:0;defc576eb6b7:33347 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T09:46:41,879 INFO [M:0;defc576eb6b7:33347 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/.lastflushedseqids 2024-11-14T09:46:41,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742414_1590 (size=72) 2024-11-14T09:46:41,891 INFO [M:0;defc576eb6b7:33347 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T09:46:41,892 INFO [M:0;defc576eb6b7:33347 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T09:46:41,892 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:46:41,892 INFO [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:46:41,892 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:46:41,892 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:46:41,892 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:46:41,892 INFO [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=751.44 KB heapSize=919.44 KB 2024-11-14T09:46:41,912 DEBUG [M:0;defc576eb6b7:33347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e95336e60244292968f45e1b522cf76 is 82, key is hbase:meta,,1/info:regioninfo/1731577412568/Put/seqid=0 2024-11-14T09:46:41,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742415_1591 (size=5672) 2024-11-14T09:46:41,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:46:41,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42511-0x10138ba60a10001, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:46:41,957 INFO [RS:0;defc576eb6b7:42511 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:46:41,958 INFO [RS:0;defc576eb6b7:42511 {}] regionserver.HRegionServer(1031): Exiting; stopping=defc576eb6b7,42511,1731577408298; zookeeper connection closed. 2024-11-14T09:46:41,958 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ec0c30 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ec0c30 2024-11-14T09:46:41,959 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T09:46:42,315 INFO [M:0;defc576eb6b7:33347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2082 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e95336e60244292968f45e1b522cf76 2024-11-14T09:46:42,342 DEBUG [M:0;defc576eb6b7:33347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9cc7fa54ece44bd9ba11c30e72bc23db is 2479, key is \x00\x00\x00\x00\x00\x00\x00R/proc:d/1731577505657/Put/seqid=0 2024-11-14T09:46:42,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742416_1592 (size=42220) 2024-11-14T09:46:42,347 INFO [M:0;defc576eb6b7:33347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750.89 KB at sequenceid=2082 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9cc7fa54ece44bd9ba11c30e72bc23db 2024-11-14T09:46:42,350 INFO [M:0;defc576eb6b7:33347 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9cc7fa54ece44bd9ba11c30e72bc23db 2024-11-14T09:46:42,364 DEBUG [M:0;defc576eb6b7:33347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84211dcf146a421c8078775813fb66cc is 69, key is defc576eb6b7,42511,1731577408298/rs:state/1731577411186/Put/seqid=0 2024-11-14T09:46:42,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073742417_1593 (size=5156) 2024-11-14T09:46:42,767 INFO [M:0;defc576eb6b7:33347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2082 (bloomFilter=true), to=hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84211dcf146a421c8078775813fb66cc 2024-11-14T09:46:42,771 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1e95336e60244292968f45e1b522cf76 as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1e95336e60244292968f45e1b522cf76 2024-11-14T09:46:42,774 INFO [M:0;defc576eb6b7:33347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1e95336e60244292968f45e1b522cf76, entries=8, sequenceid=2082, filesize=5.5 K 2024-11-14T09:46:42,775 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9cc7fa54ece44bd9ba11c30e72bc23db as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9cc7fa54ece44bd9ba11c30e72bc23db 2024-11-14T09:46:42,778 INFO [M:0;defc576eb6b7:33347 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9cc7fa54ece44bd9ba11c30e72bc23db 2024-11-14T09:46:42,778 INFO [M:0;defc576eb6b7:33347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9cc7fa54ece44bd9ba11c30e72bc23db, entries=152, sequenceid=2082, filesize=41.2 K 2024-11-14T09:46:42,779 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/84211dcf146a421c8078775813fb66cc as hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84211dcf146a421c8078775813fb66cc 2024-11-14T09:46:42,785 INFO [M:0;defc576eb6b7:33347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35709/user/jenkins/test-data/0e0c5b2d-eb72-c301-c3db-239c06ab0e6e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/84211dcf146a421c8078775813fb66cc, entries=1, sequenceid=2082, filesize=5.0 K 2024-11-14T09:46:42,786 INFO [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(3140): Finished flush of dataSize ~751.44 KB/769476, heapSize ~919.14 KB/941200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 894ms, sequenceid=2082, compaction requested=false 2024-11-14T09:46:42,796 INFO [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:46:42,796 DEBUG [M:0;defc576eb6b7:33347 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731577601892Disabling compacts and flushes for region at 1731577601892Disabling writes for close at 1731577601892Obtaining lock to block concurrent updates at 1731577601892Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731577601892Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=769476, getHeapSize=941440, getOffHeapSize=0, getCellsCount=2259 at 1731577601892Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731577601893 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731577601893Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731577601911 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731577601911Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731577602321 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731577602341 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731577602342 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731577602351 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731577602363 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731577602363Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79db6c3e: reopening flushed file at 1731577602770 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b88ffd2: reopening flushed file at 1731577602774 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24ec7dc4: reopening flushed file at 1731577602778 (+4 ms)Finished flush of dataSize ~751.44 KB/769476, heapSize ~919.14 KB/941200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 894ms, sequenceid=2082, compaction requested=false at 1731577602786 (+8 ms)Writing region close event to WAL at 1731577602796 (+10 ms)Closed at 1731577602796 2024-11-14T09:46:42,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43879 is added to blk_1073741830_1006 (size=905933) 2024-11-14T09:46:42,802 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:46:42,802 INFO [M:0;defc576eb6b7:33347 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T09:46:42,802 INFO [M:0;defc576eb6b7:33347 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33347 2024-11-14T09:46:42,803 INFO [M:0;defc576eb6b7:33347 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:46:42,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:46:42,914 INFO [M:0;defc576eb6b7:33347 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:46:42,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33347-0x10138ba60a10000, quorum=127.0.0.1:60372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:46:42,920 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:46:42,922 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:46:42,922 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:46:42,922 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:46:42,922 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/hadoop.log.dir/,STOPPED} 2024-11-14T09:46:42,925 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T09:46:42,925 WARN [BP-1223621905-172.17.0.2-1731577403708 heartbeating to localhost/127.0.0.1:35709 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T09:46:42,925 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T09:46:42,925 WARN [BP-1223621905-172.17.0.2-1731577403708 heartbeating to localhost/127.0.0.1:35709 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1223621905-172.17.0.2-1731577403708 (Datanode Uuid f95f21a8-38fa-4632-ae8d-7e262c9fd601) service to localhost/127.0.0.1:35709 2024-11-14T09:46:42,929 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/data/data1/current/BP-1223621905-172.17.0.2-1731577403708 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:46:42,929 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/cluster_90388c29-bd0a-ce19-63af-4ca2acedf7fd/data/data2/current/BP-1223621905-172.17.0.2-1731577403708 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:46:42,929 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T09:46:42,938 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:46:42,938 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:46:42,938 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:46:42,938 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:46:42,939 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bab4f5af-d205-71fe-a4bb-7ba8fcf5f197/hadoop.log.dir/,STOPPED} 2024-11-14T09:46:42,954 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T09:46:43,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down